drm/ttm: fix two regressions since move_notify changes
[linux-2.6.git] / mm / memcontrol.c
index 0356cb6..3dbff4d 100644 (file)
@@ -33,6 +33,7 @@
 #include <linux/bit_spinlock.h>
 #include <linux/rcupdate.h>
 #include <linux/limits.h>
+#include <linux/export.h>
 #include <linux/mutex.h>
 #include <linux/rbtree.h>
 #include <linux/slab.h>
@@ -49,6 +50,8 @@
 #include <linux/cpu.h>
 #include <linux/oom.h>
 #include "internal.h"
+#include <net/sock.h>
+#include <net/tcp_memcontrol.h>
 
 #include <asm/uaccess.h>
 
@@ -73,15 +76,6 @@ static int really_do_swap_account __initdata = 0;
 #define do_swap_account                (0)
 #endif
 
-/*
- * Per memcg event counter is incremented at every pagein/pageout. This counter
- * is used for trigger some periodic events. This is straightforward and better
- * than using jiffies etc. to handle periodic memcg event.
- *
- * These values will be used as !((event) & ((1 <<(thresh)) - 1))
- */
-#define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
-#define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
 
 /*
  * Statistics for memory cgroup.
@@ -93,31 +87,58 @@ enum mem_cgroup_stat_index {
        MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
        MEM_CGROUP_STAT_RSS,       /* # of pages charged as anon rss */
        MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
-       MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
-       MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
        MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
        MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
-       /* incremented at every  pagein/pageout */
-       MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
        MEM_CGROUP_ON_MOVE,     /* someone is moving account between groups */
-
        MEM_CGROUP_STAT_NSTATS,
 };
 
+enum mem_cgroup_events_index {
+       MEM_CGROUP_EVENTS_PGPGIN,       /* # of pages paged in */
+       MEM_CGROUP_EVENTS_PGPGOUT,      /* # of pages paged out */
+       MEM_CGROUP_EVENTS_COUNT,        /* # of pages paged in/out */
+       MEM_CGROUP_EVENTS_PGFAULT,      /* # of page-faults */
+       MEM_CGROUP_EVENTS_PGMAJFAULT,   /* # of major page-faults */
+       MEM_CGROUP_EVENTS_NSTATS,
+};
+/*
+ * Per memcg event counter is incremented at every pagein/pageout. With THP,
+ * it will be incremated by the number of pages. This counter is used for
+ * for trigger some periodic events. This is straightforward and better
+ * than using jiffies etc. to handle periodic memcg event.
+ */
+enum mem_cgroup_events_target {
+       MEM_CGROUP_TARGET_THRESH,
+       MEM_CGROUP_TARGET_SOFTLIMIT,
+       MEM_CGROUP_TARGET_NUMAINFO,
+       MEM_CGROUP_NTARGETS,
+};
+#define THRESHOLDS_EVENTS_TARGET (128)
+#define SOFTLIMIT_EVENTS_TARGET (1024)
+#define NUMAINFO_EVENTS_TARGET (1024)
+
 struct mem_cgroup_stat_cpu {
-       s64 count[MEM_CGROUP_STAT_NSTATS];
+       long count[MEM_CGROUP_STAT_NSTATS];
+       unsigned long events[MEM_CGROUP_EVENTS_NSTATS];
+       unsigned long targets[MEM_CGROUP_NTARGETS];
+};
+
+struct mem_cgroup_reclaim_iter {
+       /* css_id of the last scanned hierarchy member */
+       int position;
+       /* scan generation, increased every round-trip */
+       unsigned int generation;
 };
 
 /*
  * per-zone information in memory controller.
  */
 struct mem_cgroup_per_zone {
-       /*
-        * spin_lock to protect the per cgroup LRU
-        */
-       struct list_head        lists[NR_LRU_LISTS];
+       struct lruvec           lruvec;
        unsigned long           count[NR_LRU_LISTS];
 
+       struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
+
        struct zone_reclaim_stat reclaim_stat;
        struct rb_node          tree_node;      /* RB tree node */
        unsigned long long      usage_in_excess;/* Set to the value by which */
@@ -189,8 +210,8 @@ struct mem_cgroup_eventfd_list {
        struct eventfd_ctx *eventfd;
 };
 
-static void mem_cgroup_threshold(struct mem_cgroup *mem);
-static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
+static void mem_cgroup_threshold(struct mem_cgroup *memcg);
+static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
 
 /*
  * The memory controller data structure. The memory controller controls both
@@ -218,25 +239,23 @@ struct mem_cgroup {
         * per zone LRU lists.
         */
        struct mem_cgroup_lru_info info;
-
-       /*
-         protect against reclaim related member.
-       */
-       spinlock_t reclaim_param_lock;
-
-       /*
-        * While reclaiming in a hierarchy, we cache the last child we
-        * reclaimed from.
-        */
-       int last_scanned_child;
+       int last_scanned_node;
+#if MAX_NUMNODES > 1
+       nodemask_t      scan_nodes;
+       atomic_t        numainfo_events;
+       atomic_t        numainfo_updating;
+#endif
        /*
         * Should the accounting and control be hierarchical, per subtree?
         */
        bool use_hierarchy;
-       atomic_t        oom_lock;
+
+       bool            oom_lock;
+       atomic_t        under_oom;
+
        atomic_t        refcnt;
 
-       unsigned int    swappiness;
+       int     swappiness;
        /* OOM-Killer disable */
        int             oom_kill_disable;
 
@@ -270,6 +289,10 @@ struct mem_cgroup {
         */
        struct mem_cgroup_stat_cpu nocpu_base;
        spinlock_t pcp_counter_lock;
+
+#ifdef CONFIG_INET
+       struct tcp_memcontrol tcp_mem;
+#endif
 };
 
 /* Stuffs for move charges at task migration. */
@@ -344,36 +367,90 @@ enum charge_type {
 #define MEM_CGROUP_RECLAIM_NOSWAP      (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
 #define MEM_CGROUP_RECLAIM_SHRINK_BIT  0x1
 #define MEM_CGROUP_RECLAIM_SHRINK      (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
-#define MEM_CGROUP_RECLAIM_SOFT_BIT    0x2
-#define MEM_CGROUP_RECLAIM_SOFT                (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
 
-static void mem_cgroup_get(struct mem_cgroup *mem);
-static void mem_cgroup_put(struct mem_cgroup *mem);
-static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
-static void drain_all_stock_async(void);
+static void mem_cgroup_get(struct mem_cgroup *memcg);
+static void mem_cgroup_put(struct mem_cgroup *memcg);
 
-static struct mem_cgroup_per_zone *
-mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
+/* Writing them here to avoid exposing memcg's inner layout */
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
+#include <net/sock.h>
+#include <net/ip.h>
+
+static bool mem_cgroup_is_root(struct mem_cgroup *memcg);
+void sock_update_memcg(struct sock *sk)
 {
-       return &mem->info.nodeinfo[nid]->zoneinfo[zid];
+       if (static_branch(&memcg_socket_limit_enabled)) {
+               struct mem_cgroup *memcg;
+
+               BUG_ON(!sk->sk_prot->proto_cgroup);
+
+               /* Socket cloning can throw us here with sk_cgrp already
+                * filled. It won't however, necessarily happen from
+                * process context. So the test for root memcg given
+                * the current task's memcg won't help us in this case.
+                *
+                * Respecting the original socket's memcg is a better
+                * decision in this case.
+                */
+               if (sk->sk_cgrp) {
+                       BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
+                       mem_cgroup_get(sk->sk_cgrp->memcg);
+                       return;
+               }
+
+               rcu_read_lock();
+               memcg = mem_cgroup_from_task(current);
+               if (!mem_cgroup_is_root(memcg)) {
+                       mem_cgroup_get(memcg);
+                       sk->sk_cgrp = sk->sk_prot->proto_cgroup(memcg);
+               }
+               rcu_read_unlock();
+       }
 }
+EXPORT_SYMBOL(sock_update_memcg);
 
-struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
+void sock_release_memcg(struct sock *sk)
 {
-       return &mem->css;
+       if (static_branch(&memcg_socket_limit_enabled) && sk->sk_cgrp) {
+               struct mem_cgroup *memcg;
+               WARN_ON(!sk->sk_cgrp->memcg);
+               memcg = sk->sk_cgrp->memcg;
+               mem_cgroup_put(memcg);
+       }
+}
+
+#ifdef CONFIG_INET
+struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
+{
+       if (!memcg || mem_cgroup_is_root(memcg))
+               return NULL;
+
+       return &memcg->tcp_mem.cg_proto;
 }
+EXPORT_SYMBOL(tcp_proto_cgroup);
+#endif /* CONFIG_INET */
+#endif /* CONFIG_CGROUP_MEM_RES_CTLR_KMEM */
+
+static void drain_all_stock_async(struct mem_cgroup *memcg);
 
 static struct mem_cgroup_per_zone *
-page_cgroup_zoneinfo(struct page_cgroup *pc)
+mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
 {
-       struct mem_cgroup *mem = pc->mem_cgroup;
-       int nid = page_cgroup_nid(pc);
-       int zid = page_cgroup_zid(pc);
+       return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
+}
 
-       if (!mem)
-               return NULL;
+struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
+{
+       return &memcg->css;
+}
+
+static struct mem_cgroup_per_zone *
+page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
+{
+       int nid = page_to_nid(page);
+       int zid = page_zonenum(page);
 
-       return mem_cgroup_zoneinfo(mem, nid, zid);
+       return mem_cgroup_zoneinfo(memcg, nid, zid);
 }
 
 static struct mem_cgroup_tree_per_zone *
@@ -392,7 +469,7 @@ soft_limit_tree_from_page(struct page *page)
 }
 
 static void
-__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
+__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz,
                                unsigned long long new_usage_in_excess)
@@ -426,7 +503,7 @@ __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
 }
 
 static void
-__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz)
 {
@@ -437,17 +514,17 @@ __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
 }
 
 static void
-mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz)
 {
        spin_lock(&mctz->lock);
-       __mem_cgroup_remove_exceeded(mem, mz, mctz);
+       __mem_cgroup_remove_exceeded(memcg, mz, mctz);
        spin_unlock(&mctz->lock);
 }
 
 
-static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
+static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
 {
        unsigned long long excess;
        struct mem_cgroup_per_zone *mz;
@@ -460,9 +537,9 @@ static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
         * Necessary to update all ancestors when hierarchy is used.
         * because their event counter is not touched.
         */
-       for (; mem; mem = parent_mem_cgroup(mem)) {
-               mz = mem_cgroup_zoneinfo(mem, nid, zid);
-               excess = res_counter_soft_limit_excess(&mem->res);
+       for (; memcg; memcg = parent_mem_cgroup(memcg)) {
+               mz = mem_cgroup_zoneinfo(memcg, nid, zid);
+               excess = res_counter_soft_limit_excess(&memcg->res);
                /*
                 * We have to update the tree if mz is on RB-tree or
                 * mem is over its softlimit.
@@ -471,28 +548,28 @@ static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
                        spin_lock(&mctz->lock);
                        /* if on-tree, remove it */
                        if (mz->on_tree)
-                               __mem_cgroup_remove_exceeded(mem, mz, mctz);
+                               __mem_cgroup_remove_exceeded(memcg, mz, mctz);
                        /*
                         * Insert again. mz->usage_in_excess will be updated.
                         * If excess is 0, no tree ops.
                         */
-                       __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
+                       __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
                        spin_unlock(&mctz->lock);
                }
        }
 }
 
-static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
+static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
 {
        int node, zone;
        struct mem_cgroup_per_zone *mz;
        struct mem_cgroup_tree_per_zone *mctz;
 
-       for_each_node_state(node, N_POSSIBLE) {
+       for_each_node(node) {
                for (zone = 0; zone < MAX_NR_ZONES; zone++) {
-                       mz = mem_cgroup_zoneinfo(mem, node, zone);
+                       mz = mem_cgroup_zoneinfo(memcg, node, zone);
                        mctz = soft_limit_tree_node_zone(node, zone);
-                       mem_cgroup_remove_exceeded(mem, mz, mctz);
+                       mem_cgroup_remove_exceeded(memcg, mz, mctz);
                }
        }
 }
@@ -553,102 +630,174 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  * common workload, threashold and synchonization as vmstat[] should be
  * implemented.
  */
-static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
-               enum mem_cgroup_stat_index idx)
+static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
+                                enum mem_cgroup_stat_index idx)
 {
+       long val = 0;
        int cpu;
-       s64 val = 0;
 
        get_online_cpus();
        for_each_online_cpu(cpu)
-               val += per_cpu(mem->stat->count[idx], cpu);
+               val += per_cpu(memcg->stat->count[idx], cpu);
 #ifdef CONFIG_HOTPLUG_CPU
-       spin_lock(&mem->pcp_counter_lock);
-       val += mem->nocpu_base.count[idx];
-       spin_unlock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
+       val += memcg->nocpu_base.count[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
 #endif
        put_online_cpus();
        return val;
 }
 
-static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
+static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
+                                        bool charge)
 {
-       s64 ret;
-
-       ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
-       ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
-       return ret;
+       int val = (charge) ? 1 : -1;
+       this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
 }
 
-static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
-                                        bool charge)
+static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
+                                           enum mem_cgroup_events_index idx)
 {
-       int val = (charge) ? 1 : -1;
-       this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
+       unsigned long val = 0;
+       int cpu;
+
+       for_each_online_cpu(cpu)
+               val += per_cpu(memcg->stat->events[idx], cpu);
+#ifdef CONFIG_HOTPLUG_CPU
+       spin_lock(&memcg->pcp_counter_lock);
+       val += memcg->nocpu_base.events[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
+#endif
+       return val;
 }
 
-static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
+static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
                                         bool file, int nr_pages)
 {
        preempt_disable();
 
        if (file)
-               __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
+               __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
+                               nr_pages);
        else
-               __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
+               __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
+                               nr_pages);
 
        /* pagein of a big page is an event. So, ignore page size */
        if (nr_pages > 0)
-               __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
+               __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
        else {
-               __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
+               __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
                nr_pages = -nr_pages; /* for event */
        }
 
-       __this_cpu_add(mem->stat->count[MEM_CGROUP_EVENTS], nr_pages);
+       __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
 
        preempt_enable();
 }
 
-static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
-                                       enum lru_list idx)
+unsigned long
+mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
+                       unsigned int lru_mask)
 {
-       int nid, zid;
        struct mem_cgroup_per_zone *mz;
+       enum lru_list l;
+       unsigned long ret = 0;
+
+       mz = mem_cgroup_zoneinfo(memcg, nid, zid);
+
+       for_each_lru(l) {
+               if (BIT(l) & lru_mask)
+                       ret += MEM_CGROUP_ZSTAT(mz, l);
+       }
+       return ret;
+}
+
+static unsigned long
+mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
+                       int nid, unsigned int lru_mask)
+{
        u64 total = 0;
+       int zid;
+
+       for (zid = 0; zid < MAX_NR_ZONES; zid++)
+               total += mem_cgroup_zone_nr_lru_pages(memcg,
+                                               nid, zid, lru_mask);
 
-       for_each_online_node(nid)
-               for (zid = 0; zid < MAX_NR_ZONES; zid++) {
-                       mz = mem_cgroup_zoneinfo(mem, nid, zid);
-                       total += MEM_CGROUP_ZSTAT(mz, idx);
-               }
        return total;
 }
 
-static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
+static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
+                       unsigned int lru_mask)
 {
-       s64 val;
+       int nid;
+       u64 total = 0;
+
+       for_each_node_state(nid, N_HIGH_MEMORY)
+               total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
+       return total;
+}
 
-       val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
+static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
+                                      enum mem_cgroup_events_target target)
+{
+       unsigned long val, next;
 
-       return !(val & ((1 << event_mask_shift) - 1));
+       val = __this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]);
+       next = __this_cpu_read(memcg->stat->targets[target]);
+       /* from time_after() in jiffies.h */
+       if ((long)next - (long)val < 0) {
+               switch (target) {
+               case MEM_CGROUP_TARGET_THRESH:
+                       next = val + THRESHOLDS_EVENTS_TARGET;
+                       break;
+               case MEM_CGROUP_TARGET_SOFTLIMIT:
+                       next = val + SOFTLIMIT_EVENTS_TARGET;
+                       break;
+               case MEM_CGROUP_TARGET_NUMAINFO:
+                       next = val + NUMAINFO_EVENTS_TARGET;
+                       break;
+               default:
+                       break;
+               }
+               __this_cpu_write(memcg->stat->targets[target], next);
+               return true;
+       }
+       return false;
 }
 
 /*
  * Check events in order.
  *
  */
-static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
+static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
 {
+       preempt_disable();
        /* threshold event is triggered in finer grain than soft limit */
-       if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
-               mem_cgroup_threshold(mem);
-               if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
-                       mem_cgroup_update_tree(mem, page);
-       }
+       if (unlikely(mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_THRESH))) {
+               bool do_softlimit, do_numainfo;
+
+               do_softlimit = mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_SOFTLIMIT);
+#if MAX_NUMNODES > 1
+               do_numainfo = mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_NUMAINFO);
+#endif
+               preempt_enable();
+
+               mem_cgroup_threshold(memcg);
+               if (unlikely(do_softlimit))
+                       mem_cgroup_update_tree(memcg, page);
+#if MAX_NUMNODES > 1
+               if (unlikely(do_numainfo))
+                       atomic_inc(&memcg->numainfo_events);
+#endif
+       } else
+               preempt_enable();
 }
 
-static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
+struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
        return container_of(cgroup_subsys_state(cont,
                                mem_cgroup_subsys_id), struct mem_cgroup,
@@ -669,9 +818,9 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
                                struct mem_cgroup, css);
 }
 
-static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
+struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
 
        if (!mm)
                return NULL;
@@ -682,95 +831,176 @@ static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
         */
        rcu_read_lock();
        do {
-               mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
-               if (unlikely(!mem))
+               memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
+               if (unlikely(!memcg))
                        break;
-       } while (!css_tryget(&mem->css));
+       } while (!css_tryget(&memcg->css));
        rcu_read_unlock();
-       return mem;
+       return memcg;
 }
 
-/* The caller has to guarantee "mem" exists before calling this */
-static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
+/**
+ * mem_cgroup_iter - iterate over memory cgroup hierarchy
+ * @root: hierarchy root
+ * @prev: previously returned memcg, NULL on first invocation
+ * @reclaim: cookie for shared reclaim walks, NULL for full walks
+ *
+ * Returns references to children of the hierarchy below @root, or
+ * @root itself, or %NULL after a full round-trip.
+ *
+ * Caller must pass the return value in @prev on subsequent
+ * invocations for reference counting, or use mem_cgroup_iter_break()
+ * to cancel a hierarchy walk before the round-trip is complete.
+ *
+ * Reclaimers can specify a zone and a priority level in @reclaim to
+ * divide up the memcgs in the hierarchy among all concurrent
+ * reclaimers operating on the same zone and priority.
+ */
+struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
+                                  struct mem_cgroup *prev,
+                                  struct mem_cgroup_reclaim_cookie *reclaim)
 {
-       struct cgroup_subsys_state *css;
-       int found;
+       struct mem_cgroup *memcg = NULL;
+       int id = 0;
 
-       if (!mem) /* ROOT cgroup has the smallest ID */
-               return root_mem_cgroup; /*css_put/get against root is ignored*/
-       if (!mem->use_hierarchy) {
-               if (css_tryget(&mem->css))
-                       return mem;
+       if (mem_cgroup_disabled())
                return NULL;
-       }
-       rcu_read_lock();
-       /*
-        * searching a memory cgroup which has the smallest ID under given
-        * ROOT cgroup. (ID >= 1)
-        */
-       css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
-       if (css && css_tryget(css))
-               mem = container_of(css, struct mem_cgroup, css);
-       else
-               mem = NULL;
-       rcu_read_unlock();
-       return mem;
-}
 
-static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
-                                       struct mem_cgroup *root,
-                                       bool cond)
-{
-       int nextid = css_id(&iter->css) + 1;
-       int found;
-       int hierarchy_used;
-       struct cgroup_subsys_state *css;
+       if (!root)
+               root = root_mem_cgroup;
 
-       hierarchy_used = iter->use_hierarchy;
+       if (prev && !reclaim)
+               id = css_id(&prev->css);
 
-       css_put(&iter->css);
-       /* If no ROOT, walk all, ignore hierarchy */
-       if (!cond || (root && !hierarchy_used))
-               return NULL;
+       if (prev && prev != root)
+               css_put(&prev->css);
 
-       if (!root)
-               root = root_mem_cgroup;
+       if (!root->use_hierarchy && root != root_mem_cgroup) {
+               if (prev)
+                       return NULL;
+               return root;
+       }
 
-       do {
-               iter = NULL;
-               rcu_read_lock();
+       while (!memcg) {
+               struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
+               struct cgroup_subsys_state *css;
+
+               if (reclaim) {
+                       int nid = zone_to_nid(reclaim->zone);
+                       int zid = zone_idx(reclaim->zone);
+                       struct mem_cgroup_per_zone *mz;
+
+                       mz = mem_cgroup_zoneinfo(root, nid, zid);
+                       iter = &mz->reclaim_iter[reclaim->priority];
+                       if (prev && reclaim->generation != iter->generation)
+                               return NULL;
+                       id = iter->position;
+               }
 
-               css = css_get_next(&mem_cgroup_subsys, nextid,
-                               &root->css, &found);
-               if (css && css_tryget(css))
-                       iter = container_of(css, struct mem_cgroup, css);
+               rcu_read_lock();
+               css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
+               if (css) {
+                       if (css == &root->css || css_tryget(css))
+                               memcg = container_of(css,
+                                                    struct mem_cgroup, css);
+               } else
+                       id = 0;
                rcu_read_unlock();
-               /* If css is NULL, no more cgroups will be found */
-               nextid = found + 1;
-       } while (css && !iter);
 
-       return iter;
+               if (reclaim) {
+                       iter->position = id;
+                       if (!css)
+                               iter->generation++;
+                       else if (!prev && memcg)
+                               reclaim->generation = iter->generation;
+               }
+
+               if (prev && !css)
+                       return NULL;
+       }
+       return memcg;
+}
+
+/**
+ * mem_cgroup_iter_break - abort a hierarchy walk prematurely
+ * @root: hierarchy root
+ * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
+ */
+void mem_cgroup_iter_break(struct mem_cgroup *root,
+                          struct mem_cgroup *prev)
+{
+       if (!root)
+               root = root_mem_cgroup;
+       if (prev && prev != root)
+               css_put(&prev->css);
 }
+
 /*
- * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
- * be careful that "break" loop is not allowed. We have reference count.
- * Instead of that modify "cond" to be false and "continue" to exit the loop.
+ * Iteration constructs for visiting all cgroups (under a tree).  If
+ * loops are exited prematurely (break), mem_cgroup_iter_break() must
+ * be used for reference counting.
  */
-#define for_each_mem_cgroup_tree_cond(iter, root, cond)        \
-       for (iter = mem_cgroup_start_loop(root);\
-            iter != NULL;\
-            iter = mem_cgroup_get_next(iter, root, cond))
+#define for_each_mem_cgroup_tree(iter, root)           \
+       for (iter = mem_cgroup_iter(root, NULL, NULL);  \
+            iter != NULL;                              \
+            iter = mem_cgroup_iter(root, iter, NULL))
 
-#define for_each_mem_cgroup_tree(iter, root) \
-       for_each_mem_cgroup_tree_cond(iter, root, true)
+#define for_each_mem_cgroup(iter)                      \
+       for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
+            iter != NULL;                              \
+            iter = mem_cgroup_iter(NULL, iter, NULL))
 
-#define for_each_mem_cgroup_all(iter) \
-       for_each_mem_cgroup_tree_cond(iter, NULL, true)
+static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
+{
+       return (memcg == root_mem_cgroup);
+}
 
+void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
+{
+       struct mem_cgroup *memcg;
+
+       if (!mm)
+               return;
 
-static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
+       rcu_read_lock();
+       memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
+       if (unlikely(!memcg))
+               goto out;
+
+       switch (idx) {
+       case PGFAULT:
+               this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
+               break;
+       case PGMAJFAULT:
+               this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
+               break;
+       default:
+               BUG();
+       }
+out:
+       rcu_read_unlock();
+}
+EXPORT_SYMBOL(mem_cgroup_count_vm_event);
+
+/**
+ * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
+ * @zone: zone of the wanted lruvec
+ * @mem: memcg of the wanted lruvec
+ *
+ * Returns the lru list vector holding pages for the given @zone and
+ * @mem.  This can be the global zone lruvec, if the memory controller
+ * is disabled.
+ */
+struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
+                                     struct mem_cgroup *memcg)
 {
-       return (mem == root_mem_cgroup);
+       struct mem_cgroup_per_zone *mz;
+
+       if (mem_cgroup_disabled())
+               return &zone->lruvec;
+
+       mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
+       return &mz->lruvec;
 }
 
 /*
@@ -787,186 +1017,156 @@ static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
  * When moving account, the page is not on LRU. It's isolated.
  */
 
-void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
-{
-       struct page_cgroup *pc;
-       struct mem_cgroup_per_zone *mz;
-
-       if (mem_cgroup_disabled())
-               return;
-       pc = lookup_page_cgroup(page);
-       /* can happen while we handle swapcache. */
-       if (!TestClearPageCgroupAcctLRU(pc))
-               return;
-       VM_BUG_ON(!pc->mem_cgroup);
-       /*
-        * We don't check PCG_USED bit. It's cleared when the "page" is finally
-        * removed from global LRU.
-        */
-       mz = page_cgroup_zoneinfo(pc);
-       /* huge page split is done under lru_lock. so, we have no races. */
-       MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       VM_BUG_ON(list_empty(&pc->lru));
-       list_del_init(&pc->lru);
-}
-
-void mem_cgroup_del_lru(struct page *page)
-{
-       mem_cgroup_del_lru_list(page, page_lru(page));
-}
-
-/*
- * Writeback is about to end against a page which has been marked for immediate
- * reclaim.  If it still appears to be reclaimable, move it to the tail of the
- * inactive list.
+/**
+ * mem_cgroup_lru_add_list - account for adding an lru page and return lruvec
+ * @zone: zone of the page
+ * @page: the page
+ * @lru: current lru
+ *
+ * This function accounts for @page being added to @lru, and returns
+ * the lruvec for the given @zone and the memcg @page is charged to.
+ *
+ * The callsite is then responsible for physically linking the page to
+ * the returned lruvec->lists[@lru].
  */
-void mem_cgroup_rotate_reclaimable_page(struct page *page)
+struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
+                                      enum lru_list lru)
 {
        struct mem_cgroup_per_zone *mz;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc;
-       enum lru_list lru = page_lru(page);
 
        if (mem_cgroup_disabled())
-               return;
+               return &zone->lruvec;
 
        pc = lookup_page_cgroup(page);
-       /* unused or root page is not rotated. */
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       mz = page_cgroup_zoneinfo(pc);
-       list_move_tail(&pc->lru, &mz->lists[lru]);
+       memcg = pc->mem_cgroup;
+       mz = page_cgroup_zoneinfo(memcg, page);
+       /* compound_order() is stabilized through lru_lock */
+       MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
+       return &mz->lruvec;
 }
 
-void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
+/**
+ * mem_cgroup_lru_del_list - account for removing an lru page
+ * @page: the page
+ * @lru: target lru
+ *
+ * This function accounts for @page being removed from @lru.
+ *
+ * The callsite is then responsible for physically unlinking
+ * @page->lru.
+ */
+void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
 {
        struct mem_cgroup_per_zone *mz;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc;
 
        if (mem_cgroup_disabled())
                return;
 
        pc = lookup_page_cgroup(page);
-       /* unused or root page is not rotated. */
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       mz = page_cgroup_zoneinfo(pc);
-       list_move(&pc->lru, &mz->lists[lru]);
+       memcg = pc->mem_cgroup;
+       VM_BUG_ON(!memcg);
+       mz = page_cgroup_zoneinfo(memcg, page);
+       /* huge page split is done under lru_lock. so, we have no races. */
+       VM_BUG_ON(MEM_CGROUP_ZSTAT(mz, lru) < (1 << compound_order(page)));
+       MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
 }
 
-void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
+void mem_cgroup_lru_del(struct page *page)
 {
-       struct page_cgroup *pc;
-       struct mem_cgroup_per_zone *mz;
-
-       if (mem_cgroup_disabled())
-               return;
-       pc = lookup_page_cgroup(page);
-       VM_BUG_ON(PageCgroupAcctLRU(pc));
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       mz = page_cgroup_zoneinfo(pc);
-       /* huge page split is done under lru_lock. so, we have no races. */
-       MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
-       SetPageCgroupAcctLRU(pc);
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       list_add(&pc->lru, &mz->lists[lru]);
+       mem_cgroup_lru_del_list(page, page_lru(page));
 }
 
-/*
- * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
- * lru because the page may.be reused after it's fully uncharged (because of
- * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
- * it again. This function is only used to charge SwapCache. It's done under
- * lock_page and expected that zone->lru_lock is never held.
+/**
+ * mem_cgroup_lru_move_lists - account for moving a page between lrus
+ * @zone: zone of the page
+ * @page: the page
+ * @from: current lru
+ * @to: target lru
+ *
+ * This function accounts for @page being moved between the lrus @from
+ * and @to, and returns the lruvec for the given @zone and the memcg
+ * @page is charged to.
+ *
+ * The callsite is then responsible for physically relinking
+ * @page->lru to the returned lruvec->lists[@to].
  */
-static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
+struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
+                                        struct page *page,
+                                        enum lru_list from,
+                                        enum lru_list to)
 {
-       unsigned long flags;
-       struct zone *zone = page_zone(page);
-       struct page_cgroup *pc = lookup_page_cgroup(page);
-
-       spin_lock_irqsave(&zone->lru_lock, flags);
-       /*
-        * Forget old LRU when this page_cgroup is *not* used. This Used bit
-        * is guarded by lock_page() because the page is SwapCache.
-        */
-       if (!PageCgroupUsed(pc))
-               mem_cgroup_del_lru_list(page, page_lru(page));
-       spin_unlock_irqrestore(&zone->lru_lock, flags);
+       /* XXX: Optimize this, especially for @from == @to */
+       mem_cgroup_lru_del_list(page, from);
+       return mem_cgroup_lru_add_list(zone, page, to);
 }
 
-static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
+/*
+ * Checks whether given mem is same or in the root_mem_cgroup's
+ * hierarchy subtree
+ */
+static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
+               struct mem_cgroup *memcg)
 {
-       unsigned long flags;
-       struct zone *zone = page_zone(page);
-       struct page_cgroup *pc = lookup_page_cgroup(page);
-
-       spin_lock_irqsave(&zone->lru_lock, flags);
-       /* link when the page is linked to LRU but page_cgroup isn't */
-       if (PageLRU(page) && !PageCgroupAcctLRU(pc))
-               mem_cgroup_add_lru_list(page, page_lru(page));
-       spin_unlock_irqrestore(&zone->lru_lock, flags);
-}
-
+       if (root_memcg != memcg) {
+               return (root_memcg->use_hierarchy &&
+                       css_is_ancestor(&memcg->css, &root_memcg->css));
+       }
 
-void mem_cgroup_move_lists(struct page *page,
-                          enum lru_list from, enum lru_list to)
-{
-       if (mem_cgroup_disabled())
-               return;
-       mem_cgroup_del_lru_list(page, from);
-       mem_cgroup_add_lru_list(page, to);
+       return true;
 }
 
-int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
+int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
 {
        int ret;
        struct mem_cgroup *curr = NULL;
        struct task_struct *p;
 
        p = find_lock_task_mm(task);
-       if (!p)
-               return 0;
-       curr = try_get_mem_cgroup_from_mm(p->mm);
-       task_unlock(p);
+       if (p) {
+               curr = try_get_mem_cgroup_from_mm(p->mm);
+               task_unlock(p);
+       } else {
+               /*
+                * All threads may have already detached their mm's, but the oom
+                * killer still needs to detect if they have already been oom
+                * killed to prevent needlessly killing additional tasks.
+                */
+               task_lock(task);
+               curr = mem_cgroup_from_task(task);
+               if (curr)
+                       css_get(&curr->css);
+               task_unlock(task);
+       }
        if (!curr)
                return 0;
        /*
-        * We should check use_hierarchy of "mem" not "curr". Because checking
+        * We should check use_hierarchy of "memcg" not "curr". Because checking
         * use_hierarchy of "curr" here make this function true if hierarchy is
-        * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
-        * hierarchy(even if use_hierarchy is disabled in "mem").
+        * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
+        * hierarchy(even if use_hierarchy is disabled in "memcg").
         */
-       if (mem->use_hierarchy)
-               ret = css_is_ancestor(&curr->css, &mem->css);
-       else
-               ret = (curr == mem);
+       ret = mem_cgroup_same_or_subtree(memcg, curr);
        css_put(&curr->css);
        return ret;
 }
 
-static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
+int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
 {
-       unsigned long active;
+       unsigned long inactive_ratio;
+       int nid = zone_to_nid(zone);
+       int zid = zone_idx(zone);
        unsigned long inactive;
+       unsigned long active;
        unsigned long gb;
-       unsigned long inactive_ratio;
 
-       inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
-       active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
+       inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                               BIT(LRU_INACTIVE_ANON));
+       active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                             BIT(LRU_ACTIVE_ANON));
 
        gb = (inactive + active) >> (30 - PAGE_SHIFT);
        if (gb)
@@ -974,54 +1174,24 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_
        else
                inactive_ratio = 1;
 
-       if (present_pages) {
-               present_pages[0] = inactive;
-               present_pages[1] = active;
-       }
-
-       return inactive_ratio;
-}
-
-int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
-{
-       unsigned long active;
-       unsigned long inactive;
-       unsigned long present_pages[2];
-       unsigned long inactive_ratio;
-
-       inactive_ratio = calc_inactive_ratio(memcg, present_pages);
-
-       inactive = present_pages[0];
-       active = present_pages[1];
-
-       if (inactive * inactive_ratio < active)
-               return 1;
-
-       return 0;
+       return inactive * inactive_ratio < active;
 }
 
-int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
+int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
 {
        unsigned long active;
        unsigned long inactive;
+       int zid = zone_idx(zone);
+       int nid = zone_to_nid(zone);
 
-       inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
-       active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
+       inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                               BIT(LRU_INACTIVE_FILE));
+       active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                             BIT(LRU_ACTIVE_FILE));
 
        return (active > inactive);
 }
 
-unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
-                                      struct zone *zone,
-                                      enum lru_list lru)
-{
-       int nid = zone_to_nid(zone);
-       int zid = zone_idx(zone);
-       struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
-
-       return MEM_CGROUP_ZSTAT(mz, lru);
-}
-
 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
                                                      struct zone *zone)
 {
@@ -1046,72 +1216,10 @@ mem_cgroup_get_reclaim_stat_from_page(struct page *page)
                return NULL;
        /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
        smp_rmb();
-       mz = page_cgroup_zoneinfo(pc);
-       if (!mz)
-               return NULL;
-
+       mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
        return &mz->reclaim_stat;
 }
 
-unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
-                                       struct list_head *dst,
-                                       unsigned long *scanned, int order,
-                                       int mode, struct zone *z,
-                                       struct mem_cgroup *mem_cont,
-                                       int active, int file)
-{
-       unsigned long nr_taken = 0;
-       struct page *page;
-       unsigned long scan;
-       LIST_HEAD(pc_list);
-       struct list_head *src;
-       struct page_cgroup *pc, *tmp;
-       int nid = zone_to_nid(z);
-       int zid = zone_idx(z);
-       struct mem_cgroup_per_zone *mz;
-       int lru = LRU_FILE * file + active;
-       int ret;
-
-       BUG_ON(!mem_cont);
-       mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
-       src = &mz->lists[lru];
-
-       scan = 0;
-       list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
-               if (scan >= nr_to_scan)
-                       break;
-
-               page = pc->page;
-               if (unlikely(!PageCgroupUsed(pc)))
-                       continue;
-               if (unlikely(!PageLRU(page)))
-                       continue;
-
-               scan++;
-               ret = __isolate_lru_page(page, mode, file);
-               switch (ret) {
-               case 0:
-                       list_move(&page->lru, dst);
-                       mem_cgroup_del_lru(page);
-                       nr_taken += hpage_nr_pages(page);
-                       break;
-               case -EBUSY:
-                       /* we don't affect global LRU but rotate in our LRU */
-                       mem_cgroup_rotate_lru_list(page, page_lru(page));
-                       break;
-               default:
-                       break;
-               }
-       }
-
-       *scanned = scan;
-
-       trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
-                                     0, 0, 0, mode);
-
-       return nr_taken;
-}
-
 #define mem_cgroup_from_res_counter(counter, member)   \
        container_of(counter, struct mem_cgroup, member)
 
@@ -1120,61 +1228,56 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
  * @mem: the memory cgroup
  *
  * Returns the maximum amount of memory @mem can be charged with, in
- * bytes.
+ * pages.
  */
-static unsigned long long mem_cgroup_margin(struct mem_cgroup *mem)
+static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
 {
        unsigned long long margin;
 
-       margin = res_counter_margin(&mem->res);
+       margin = res_counter_margin(&memcg->res);
        if (do_swap_account)
-               margin = min(margin, res_counter_margin(&mem->memsw));
-       return margin;
+               margin = min(margin, res_counter_margin(&memcg->memsw));
+       return margin >> PAGE_SHIFT;
 }
 
-static unsigned int get_swappiness(struct mem_cgroup *memcg)
+int mem_cgroup_swappiness(struct mem_cgroup *memcg)
 {
        struct cgroup *cgrp = memcg->css.cgroup;
-       unsigned int swappiness;
 
        /* root ? */
        if (cgrp->parent == NULL)
                return vm_swappiness;
 
-       spin_lock(&memcg->reclaim_param_lock);
-       swappiness = memcg->swappiness;
-       spin_unlock(&memcg->reclaim_param_lock);
-
-       return swappiness;
+       return memcg->swappiness;
 }
 
-static void mem_cgroup_start_move(struct mem_cgroup *mem)
+static void mem_cgroup_start_move(struct mem_cgroup *memcg)
 {
        int cpu;
 
        get_online_cpus();
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for_each_online_cpu(cpu)
-               per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
-       mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
-       spin_unlock(&mem->pcp_counter_lock);
+               per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
+       memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
+       spin_unlock(&memcg->pcp_counter_lock);
        put_online_cpus();
 
        synchronize_rcu();
 }
 
-static void mem_cgroup_end_move(struct mem_cgroup *mem)
+static void mem_cgroup_end_move(struct mem_cgroup *memcg)
 {
        int cpu;
 
-       if (!mem)
+       if (!memcg)
                return;
        get_online_cpus();
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for_each_online_cpu(cpu)
-               per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
-       mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
-       spin_unlock(&mem->pcp_counter_lock);
+               per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
+       memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
+       spin_unlock(&memcg->pcp_counter_lock);
        put_online_cpus();
 }
 /*
@@ -1189,13 +1292,13 @@ static void mem_cgroup_end_move(struct mem_cgroup *mem)
  *                       waiting at hith-memory prressure caused by "move".
  */
 
-static bool mem_cgroup_stealed(struct mem_cgroup *mem)
+static bool mem_cgroup_stealed(struct mem_cgroup *memcg)
 {
        VM_BUG_ON(!rcu_read_lock_held());
-       return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
+       return this_cpu_read(memcg->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
 }
 
-static bool mem_cgroup_under_move(struct mem_cgroup *mem)
+static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *from;
        struct mem_cgroup *to;
@@ -1209,19 +1312,18 @@ static bool mem_cgroup_under_move(struct mem_cgroup *mem)
        to = mc.to;
        if (!from)
                goto unlock;
-       if (from == mem || to == mem
-           || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
-           || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
-               ret = true;
+
+       ret = mem_cgroup_same_or_subtree(memcg, from)
+               || mem_cgroup_same_or_subtree(memcg, to);
 unlock:
        spin_unlock(&mc.lock);
        return ret;
 }
 
-static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
+static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
 {
        if (mc.moving_task && current != mc.moving_task) {
-               if (mem_cgroup_under_move(mem)) {
+               if (mem_cgroup_under_move(memcg)) {
                        DEFINE_WAIT(wait);
                        prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
                        /* moving charge context might have finished. */
@@ -1305,12 +1407,12 @@ done:
  * This function returns the number of memcg under hierarchy tree. Returns
  * 1(self count) if no children.
  */
-static int mem_cgroup_count_children(struct mem_cgroup *mem)
+static int mem_cgroup_count_children(struct mem_cgroup *memcg)
 {
        int num = 0;
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                num++;
        return num;
 }
@@ -1334,155 +1436,292 @@ u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
        return min(limit, memsw);
 }
 
+static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
+                                       gfp_t gfp_mask,
+                                       unsigned long flags)
+{
+       unsigned long total = 0;
+       bool noswap = false;
+       int loop;
+
+       if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
+               noswap = true;
+       if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
+               noswap = true;
+
+       for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
+               if (loop)
+                       drain_all_stock_async(memcg);
+               total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
+               /*
+                * Allow limit shrinkers, which are triggered directly
+                * by userspace, to catch signals and stop reclaim
+                * after minimal progress, regardless of the margin.
+                */
+               if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
+                       break;
+               if (mem_cgroup_margin(memcg))
+                       break;
+               /*
+                * If nothing was reclaimed after two attempts, there
+                * may be no reclaimable pages in this hierarchy.
+                */
+               if (loop && !total)
+                       break;
+       }
+       return total;
+}
+
+/**
+ * test_mem_cgroup_node_reclaimable
+ * @mem: the target memcg
+ * @nid: the node ID to be checked.
+ * @noswap : specify true here if the user wants flle only information.
+ *
+ * This function returns whether the specified memcg contains any
+ * reclaimable pages on a node. Returns true if there are any reclaimable
+ * pages in the node.
+ */
+static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
+               int nid, bool noswap)
+{
+       if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
+               return true;
+       if (noswap || !total_swap_pages)
+               return false;
+       if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
+               return true;
+       return false;
+
+}
+#if MAX_NUMNODES > 1
+
 /*
- * Visit the first child (need not be the first child as per the ordering
- * of the cgroup list, since we track last_scanned_child) of @mem and use
- * that to reclaim free pages from.
+ * Always updating the nodemask is not very good - even if we have an empty
+ * list or the wrong list here, we can start from some node and traverse all
+ * nodes based on the zonelist. So update the list loosely once per 10 secs.
+ *
  */
-static struct mem_cgroup *
-mem_cgroup_select_victim(struct mem_cgroup *root_mem)
+static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
 {
-       struct mem_cgroup *ret = NULL;
-       struct cgroup_subsys_state *css;
-       int nextid, found;
+       int nid;
+       /*
+        * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
+        * pagein/pageout changes since the last update.
+        */
+       if (!atomic_read(&memcg->numainfo_events))
+               return;
+       if (atomic_inc_return(&memcg->numainfo_updating) > 1)
+               return;
 
-       if (!root_mem->use_hierarchy) {
-               css_get(&root_mem->css);
-               ret = root_mem;
-       }
+       /* make a nodemask where this memcg uses memory from */
+       memcg->scan_nodes = node_states[N_HIGH_MEMORY];
 
-       while (!ret) {
-               rcu_read_lock();
-               nextid = root_mem->last_scanned_child + 1;
-               css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
-                                  &found);
-               if (css && css_tryget(css))
-                       ret = container_of(css, struct mem_cgroup, css);
+       for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {
 
-               rcu_read_unlock();
-               /* Updates scanning parameter */
-               spin_lock(&root_mem->reclaim_param_lock);
-               if (!css) {
-                       /* this means start scan from ID:1 */
-                       root_mem->last_scanned_child = 0;
-               } else
-                       root_mem->last_scanned_child = found;
-               spin_unlock(&root_mem->reclaim_param_lock);
+               if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
+                       node_clear(nid, memcg->scan_nodes);
        }
 
-       return ret;
+       atomic_set(&memcg->numainfo_events, 0);
+       atomic_set(&memcg->numainfo_updating, 0);
 }
 
 /*
- * Scan the hierarchy if needed to reclaim memory. We remember the last child
- * we reclaimed from, so that we don't end up penalizing one child extensively
- * based on its position in the children list.
- *
- * root_mem is the original ancestor that we've been reclaim from.
+ * Selecting a node where we start reclaim from. Because what we need is just
+ * reducing usage counter, start from anywhere is O,K. Considering
+ * memory reclaim from current node, there are pros. and cons.
  *
- * We give up and return to the caller when we visit root_mem twice.
- * (other groups can be removed while we're walking....)
+ * Freeing memory from current node means freeing memory from a node which
+ * we'll use or we've used. So, it may make LRU bad. And if several threads
+ * hit limits, it will see a contention on a node. But freeing from remote
+ * node means more costs for memory reclaim because of memory latency.
  *
- * If shrink==true, for avoiding to free too much, this returns immedieately.
+ * Now, we use round-robin. Better algorithm is welcomed.
  */
-static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
-                                               struct zone *zone,
-                                               gfp_t gfp_mask,
-                                               unsigned long reclaim_options)
+int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
 {
-       struct mem_cgroup *victim;
-       int ret, total = 0;
+       int node;
+
+       mem_cgroup_may_update_nodemask(memcg);
+       node = memcg->last_scanned_node;
+
+       node = next_node(node, memcg->scan_nodes);
+       if (node == MAX_NUMNODES)
+               node = first_node(memcg->scan_nodes);
+       /*
+        * We call this when we hit limit, not when pages are added to LRU.
+        * No LRU may hold pages because all pages are UNEVICTABLE or
+        * memcg is too small and all pages are not on LRU. In that case,
+        * we use curret node.
+        */
+       if (unlikely(node == MAX_NUMNODES))
+               node = numa_node_id();
+
+       memcg->last_scanned_node = node;
+       return node;
+}
+
+/*
+ * Check all nodes whether it contains reclaimable pages or not.
+ * For quick scan, we make use of scan_nodes. This will allow us to skip
+ * unused nodes. But scan_nodes is lazily updated and may not cotain
+ * enough new information. We need to do double check.
+ */
+bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
+{
+       int nid;
+
+       /*
+        * quick check...making use of scan_node.
+        * We can skip unused nodes.
+        */
+       if (!nodes_empty(memcg->scan_nodes)) {
+               for (nid = first_node(memcg->scan_nodes);
+                    nid < MAX_NUMNODES;
+                    nid = next_node(nid, memcg->scan_nodes)) {
+
+                       if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
+                               return true;
+               }
+       }
+       /*
+        * Check rest of nodes.
+        */
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               if (node_isset(nid, memcg->scan_nodes))
+                       continue;
+               if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
+                       return true;
+       }
+       return false;
+}
+
+#else
+int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
+{
+       return 0;
+}
+
+bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
+{
+       return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
+}
+#endif
+
+static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
+                                  struct zone *zone,
+                                  gfp_t gfp_mask,
+                                  unsigned long *total_scanned)
+{
+       struct mem_cgroup *victim = NULL;
+       int total = 0;
        int loop = 0;
-       bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
-       bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
-       bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
        unsigned long excess;
+       unsigned long nr_scanned;
+       struct mem_cgroup_reclaim_cookie reclaim = {
+               .zone = zone,
+               .priority = 0,
+       };
 
-       excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT;
-
-       /* If memsw_is_minimum==1, swap-out is of-no-use. */
-       if (root_mem->memsw_is_minimum)
-               noswap = true;
+       excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
 
        while (1) {
-               victim = mem_cgroup_select_victim(root_mem);
-               if (victim == root_mem) {
+               victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
+               if (!victim) {
                        loop++;
-                       if (loop >= 1)
-                               drain_all_stock_async();
                        if (loop >= 2) {
                                /*
                                 * If we have not been able to reclaim
                                 * anything, it might because there are
                                 * no reclaimable pages under this hierarchy
                                 */
-                               if (!check_soft || !total) {
-                                       css_put(&victim->css);
+                               if (!total)
                                        break;
-                               }
                                /*
-                                * We want to do more targetted reclaim.
+                                * We want to do more targeted reclaim.
                                 * excess >> 2 is not to excessive so as to
                                 * reclaim too much, nor too less that we keep
                                 * coming back to reclaim from this cgroup
                                 */
                                if (total >= (excess >> 2) ||
-                                       (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
-                                       css_put(&victim->css);
+                                       (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
                                        break;
-                               }
                        }
-               }
-               if (!mem_cgroup_local_usage(victim)) {
-                       /* this cgroup's local usage == 0 */
-                       css_put(&victim->css);
                        continue;
                }
-               /* we use swappiness of local cgroup */
-               if (check_soft)
-                       ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
-                               noswap, get_swappiness(victim), zone);
-               else
-                       ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
-                                               noswap, get_swappiness(victim));
-               css_put(&victim->css);
-               /*
-                * At shrinking usage, we can't check we should stop here or
-                * reclaim more. It's depends on callers. last_scanned_child
-                * will work enough for keeping fairness under tree.
-                */
-               if (shrink)
-                       return ret;
-               total += ret;
-               if (check_soft) {
-                       if (!res_counter_soft_limit_excess(&root_mem->res))
-                               return total;
-               } else if (mem_cgroup_margin(root_mem))
-                       return 1 + total;
+               if (!mem_cgroup_reclaimable(victim, false))
+                       continue;
+               total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
+                                                    zone, &nr_scanned);
+               *total_scanned += nr_scanned;
+               if (!res_counter_soft_limit_excess(&root_memcg->res))
+                       break;
        }
+       mem_cgroup_iter_break(root_memcg, victim);
        return total;
 }
 
 /*
  * Check OOM-Killer is already running under our hierarchy.
  * If someone is running, return false.
+ * Has to be called with memcg_oom_lock
  */
-static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
+static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
 {
-       int x, lock_count = 0;
-       struct mem_cgroup *iter;
+       struct mem_cgroup *iter, *failed = NULL;
 
-       for_each_mem_cgroup_tree(iter, mem) {
-               x = atomic_inc_return(&iter->oom_lock);
-               lock_count = max(x, lock_count);
+       for_each_mem_cgroup_tree(iter, memcg) {
+               if (iter->oom_lock) {
+                       /*
+                        * this subtree of our hierarchy is already locked
+                        * so we cannot give a lock.
+                        */
+                       failed = iter;
+                       mem_cgroup_iter_break(memcg, iter);
+                       break;
+               } else
+                       iter->oom_lock = true;
        }
 
-       if (lock_count == 1)
+       if (!failed)
                return true;
+
+       /*
+        * OK, we failed to lock the whole subtree so we have to clean up
+        * what we set up to the failing subtree
+        */
+       for_each_mem_cgroup_tree(iter, memcg) {
+               if (iter == failed) {
+                       mem_cgroup_iter_break(memcg, iter);
+                       break;
+               }
+               iter->oom_lock = false;
+       }
        return false;
 }
 
-static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
+/*
+ * Has to be called with memcg_oom_lock
+ */
+static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
+{
+       struct mem_cgroup *iter;
+
+       for_each_mem_cgroup_tree(iter, memcg)
+               iter->oom_lock = false;
+       return 0;
+}
+
+static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
+{
+       struct mem_cgroup *iter;
+
+       for_each_mem_cgroup_tree(iter, memcg)
+               atomic_inc(&iter->under_oom);
+}
+
+static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *iter;
 
@@ -1491,13 +1730,11 @@ static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
         * mem_cgroup_oom_lock() may not be called. We have to use
         * atomic_add_unless() here.
         */
-       for_each_mem_cgroup_tree(iter, mem)
-               atomic_add_unless(&iter->oom_lock, -1, 0);
-       return 0;
+       for_each_mem_cgroup_tree(iter, memcg)
+               atomic_add_unless(&iter->under_oom, -1, 0);
 }
 
-
-static DEFINE_MUTEX(memcg_oom_mutex);
+static DEFINE_SPINLOCK(memcg_oom_lock);
 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
 
 struct oom_wait_info {
@@ -1508,85 +1745,85 @@ struct oom_wait_info {
 static int memcg_oom_wake_function(wait_queue_t *wait,
        unsigned mode, int sync, void *arg)
 {
-       struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
+       struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg,
+                         *oom_wait_memcg;
        struct oom_wait_info *oom_wait_info;
 
        oom_wait_info = container_of(wait, struct oom_wait_info, wait);
+       oom_wait_memcg = oom_wait_info->mem;
 
-       if (oom_wait_info->mem == wake_mem)
-               goto wakeup;
-       /* if no hierarchy, no match */
-       if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
-               return 0;
        /*
         * Both of oom_wait_info->mem and wake_mem are stable under us.
         * Then we can use css_is_ancestor without taking care of RCU.
         */
-       if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
-           !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
+       if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
+               && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
                return 0;
-
-wakeup:
        return autoremove_wake_function(wait, mode, sync, arg);
 }
 
-static void memcg_wakeup_oom(struct mem_cgroup *mem)
+static void memcg_wakeup_oom(struct mem_cgroup *memcg)
 {
-       /* for filtering, pass "mem" as argument. */
-       __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
+       /* for filtering, pass "memcg" as argument. */
+       __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
 }
 
-static void memcg_oom_recover(struct mem_cgroup *mem)
+static void memcg_oom_recover(struct mem_cgroup *memcg)
 {
-       if (mem && atomic_read(&mem->oom_lock))
-               memcg_wakeup_oom(mem);
+       if (memcg && atomic_read(&memcg->under_oom))
+               memcg_wakeup_oom(memcg);
 }
 
 /*
  * try to call OOM killer. returns false if we should exit memory-reclaim loop.
  */
-bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
+bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask)
 {
        struct oom_wait_info owait;
        bool locked, need_to_kill;
 
-       owait.mem = mem;
+       owait.mem = memcg;
        owait.wait.flags = 0;
        owait.wait.func = memcg_oom_wake_function;
        owait.wait.private = current;
        INIT_LIST_HEAD(&owait.wait.task_list);
        need_to_kill = true;
-       /* At first, try to OOM lock hierarchy under mem.*/
-       mutex_lock(&memcg_oom_mutex);
-       locked = mem_cgroup_oom_lock(mem);
+       mem_cgroup_mark_under_oom(memcg);
+
+       /* At first, try to OOM lock hierarchy under memcg.*/
+       spin_lock(&memcg_oom_lock);
+       locked = mem_cgroup_oom_lock(memcg);
        /*
         * Even if signal_pending(), we can't quit charge() loop without
         * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
         * under OOM is always welcomed, use TASK_KILLABLE here.
         */
        prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
-       if (!locked || mem->oom_kill_disable)
+       if (!locked || memcg->oom_kill_disable)
                need_to_kill = false;
        if (locked)
-               mem_cgroup_oom_notify(mem);
-       mutex_unlock(&memcg_oom_mutex);
+               mem_cgroup_oom_notify(memcg);
+       spin_unlock(&memcg_oom_lock);
 
        if (need_to_kill) {
                finish_wait(&memcg_oom_waitq, &owait.wait);
-               mem_cgroup_out_of_memory(mem, mask);
+               mem_cgroup_out_of_memory(memcg, mask);
        } else {
                schedule();
                finish_wait(&memcg_oom_waitq, &owait.wait);
        }
-       mutex_lock(&memcg_oom_mutex);
-       mem_cgroup_oom_unlock(mem);
-       memcg_wakeup_oom(mem);
-       mutex_unlock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
+       if (locked)
+               mem_cgroup_oom_unlock(memcg);
+       memcg_wakeup_oom(memcg);
+       spin_unlock(&memcg_oom_lock);
+
+       mem_cgroup_unmark_under_oom(memcg);
 
        if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
                return false;
        /* Give chance to dying process */
-       schedule_timeout(1);
+       schedule_timeout_uninterruptible(1);
        return true;
 }
 
@@ -1617,25 +1854,25 @@ bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
 void mem_cgroup_update_page_stat(struct page *page,
                                 enum mem_cgroup_page_stat_item idx, int val)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc = lookup_page_cgroup(page);
        bool need_unlock = false;
        unsigned long uninitialized_var(flags);
 
-       if (unlikely(!pc))
+       if (mem_cgroup_disabled())
                return;
 
        rcu_read_lock();
-       mem = pc->mem_cgroup;
-       if (unlikely(!mem || !PageCgroupUsed(pc)))
+       memcg = pc->mem_cgroup;
+       if (unlikely(!memcg || !PageCgroupUsed(pc)))
                goto out;
        /* pc->mem_cgroup is unstable ? */
-       if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
+       if (unlikely(mem_cgroup_stealed(memcg)) || PageTransHuge(page)) {
                /* take a lock against to access pc->mem_cgroup */
                move_lock_page_cgroup(pc, &flags);
                need_unlock = true;
-               mem = pc->mem_cgroup;
-               if (!mem || !PageCgroupUsed(pc))
+               memcg = pc->mem_cgroup;
+               if (!memcg || !PageCgroupUsed(pc))
                        goto out;
        }
 
@@ -1651,7 +1888,7 @@ void mem_cgroup_update_page_stat(struct page *page,
                BUG();
        }
 
-       this_cpu_add(mem->stat->count[idx], val);
+       this_cpu_add(memcg->stat->count[idx], val);
 
 out:
        if (unlikely(need_unlock))
@@ -1665,29 +1902,31 @@ EXPORT_SYMBOL(mem_cgroup_update_page_stat);
  * size of first charge trial. "32" comes from vmscan.c's magic value.
  * TODO: maybe necessary to use big numbers in big irons.
  */
-#define CHARGE_SIZE    (32 * PAGE_SIZE)
+#define CHARGE_BATCH   32U
 struct memcg_stock_pcp {
        struct mem_cgroup *cached; /* this never be root cgroup */
-       int charge;
+       unsigned int nr_pages;
        struct work_struct work;
+       unsigned long flags;
+#define FLUSHING_CACHED_CHARGE (0)
 };
 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
-static atomic_t memcg_drain_count;
+static DEFINE_MUTEX(percpu_charge_mutex);
 
 /*
- * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
+ * Try to consume stocked charge on this cpu. If success, one page is consumed
  * from local stock and true is returned. If the stock is 0 or charges from a
  * cgroup which is not current target, returns false. This stock will be
  * refilled.
  */
-static bool consume_stock(struct mem_cgroup *mem)
+static bool consume_stock(struct mem_cgroup *memcg)
 {
        struct memcg_stock_pcp *stock;
        bool ret = true;
 
        stock = &get_cpu_var(memcg_stock);
-       if (mem == stock->cached && stock->charge)
-               stock->charge -= PAGE_SIZE;
+       if (memcg == stock->cached && stock->nr_pages)
+               stock->nr_pages--;
        else /* need to call res_counter_charge */
                ret = false;
        put_cpu_var(memcg_stock);
@@ -1701,13 +1940,15 @@ static void drain_stock(struct memcg_stock_pcp *stock)
 {
        struct mem_cgroup *old = stock->cached;
 
-       if (stock->charge) {
-               res_counter_uncharge(&old->res, stock->charge);
+       if (stock->nr_pages) {
+               unsigned long bytes = stock->nr_pages * PAGE_SIZE;
+
+               res_counter_uncharge(&old->res, bytes);
                if (do_swap_account)
-                       res_counter_uncharge(&old->memsw, stock->charge);
+                       res_counter_uncharge(&old->memsw, bytes);
+               stock->nr_pages = 0;
        }
        stock->cached = NULL;
-       stock->charge = 0;
 }
 
 /*
@@ -1718,89 +1959,126 @@ static void drain_local_stock(struct work_struct *dummy)
 {
        struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
        drain_stock(stock);
+       clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
 }
 
 /*
  * Cache charges(val) which is from res_counter, to local per_cpu area.
  * This will be consumed by consume_stock() function, later.
  */
-static void refill_stock(struct mem_cgroup *mem, int val)
+static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 {
        struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
 
-       if (stock->cached != mem) { /* reset if necessary */
+       if (stock->cached != memcg) { /* reset if necessary */
                drain_stock(stock);
-               stock->cached = mem;
+               stock->cached = memcg;
        }
-       stock->charge += val;
+       stock->nr_pages += nr_pages;
        put_cpu_var(memcg_stock);
 }
 
 /*
- * Tries to drain stocked charges in other cpus. This function is asynchronous
- * and just put a work per cpu for draining localy on each cpu. Caller can
- * expects some charges will be back to res_counter later but cannot wait for
- * it.
+ * Drains all per-CPU charge caches for given root_memcg resp. subtree
+ * of the hierarchy under it. sync flag says whether we should block
+ * until the work is done.
  */
-static void drain_all_stock_async(void)
+static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
 {
-       int cpu;
-       /* This function is for scheduling "drain" in asynchronous way.
-        * The result of "drain" is not directly handled by callers. Then,
-        * if someone is calling drain, we don't have to call drain more.
-        * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
-        * there is a race. We just do loose check here.
-        */
-       if (atomic_read(&memcg_drain_count))
-               return;
+       int cpu, curcpu;
+
        /* Notify other cpus that system-wide "drain" is running */
-       atomic_inc(&memcg_drain_count);
        get_online_cpus();
+       curcpu = get_cpu();
+       for_each_online_cpu(cpu) {
+               struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
+               struct mem_cgroup *memcg;
+
+               memcg = stock->cached;
+               if (!memcg || !stock->nr_pages)
+                       continue;
+               if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
+                       continue;
+               if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
+                       if (cpu == curcpu)
+                               drain_local_stock(&stock->work);
+                       else
+                               schedule_work_on(cpu, &stock->work);
+               }
+       }
+       put_cpu();
+
+       if (!sync)
+               goto out;
+
        for_each_online_cpu(cpu) {
                struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
-               schedule_work_on(cpu, &stock->work);
+               if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
+                       flush_work(&stock->work);
        }
+out:
        put_online_cpus();
-       atomic_dec(&memcg_drain_count);
-       /* We don't wait for flush_work */
+}
+
+/*
+ * Tries to drain stocked charges in other cpus. This function is asynchronous
+ * and just put a work per cpu for draining localy on each cpu. Caller can
+ * expects some charges will be back to res_counter later but cannot wait for
+ * it.
+ */
+static void drain_all_stock_async(struct mem_cgroup *root_memcg)
+{
+       /*
+        * If someone calls draining, avoid adding more kworker runs.
+        */
+       if (!mutex_trylock(&percpu_charge_mutex))
+               return;
+       drain_all_stock(root_memcg, false);
+       mutex_unlock(&percpu_charge_mutex);
 }
 
 /* This is a synchronous drain interface. */
-static void drain_all_stock_sync(void)
+static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
 {
        /* called when force_empty is called */
-       atomic_inc(&memcg_drain_count);
-       schedule_on_each_cpu(drain_local_stock);
-       atomic_dec(&memcg_drain_count);
+       mutex_lock(&percpu_charge_mutex);
+       drain_all_stock(root_memcg, true);
+       mutex_unlock(&percpu_charge_mutex);
 }
 
 /*
  * This function drains percpu counter value from DEAD cpu and
  * move it to local cpu. Note that this function can be preempted.
  */
-static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
+static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
 {
        int i;
 
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
-               s64 x = per_cpu(mem->stat->count[i], cpu);
+               long x = per_cpu(memcg->stat->count[i], cpu);
 
-               per_cpu(mem->stat->count[i], cpu) = 0;
-               mem->nocpu_base.count[i] += x;
+               per_cpu(memcg->stat->count[i], cpu) = 0;
+               memcg->nocpu_base.count[i] += x;
+       }
+       for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
+               unsigned long x = per_cpu(memcg->stat->events[i], cpu);
+
+               per_cpu(memcg->stat->events[i], cpu) = 0;
+               memcg->nocpu_base.events[i] += x;
        }
        /* need to clear ON_MOVE value, works as a kind of lock. */
-       per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
-       spin_unlock(&mem->pcp_counter_lock);
+       per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
+       spin_unlock(&memcg->pcp_counter_lock);
 }
 
-static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
+static void synchronize_mem_cgroup_on_move(struct mem_cgroup *memcg, int cpu)
 {
        int idx = MEM_CGROUP_ON_MOVE;
 
-       spin_lock(&mem->pcp_counter_lock);
-       per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
-       spin_unlock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
+       per_cpu(memcg->stat->count[idx], cpu) = memcg->nocpu_base.count[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
 }
 
 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
@@ -1812,7 +2090,7 @@ static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
        struct mem_cgroup *iter;
 
        if ((action == CPU_ONLINE)) {
-               for_each_mem_cgroup_all(iter)
+               for_each_mem_cgroup(iter)
                        synchronize_mem_cgroup_on_move(iter, cpu);
                return NOTIFY_OK;
        }
@@ -1820,7 +2098,7 @@ static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
        if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
                return NOTIFY_OK;
 
-       for_each_mem_cgroup_all(iter)
+       for_each_mem_cgroup(iter)
                mem_cgroup_drain_pcp_counter(iter, cpu);
 
        stock = &per_cpu(memcg_stock, cpu);
@@ -1838,45 +2116,44 @@ enum {
        CHARGE_OOM_DIE,         /* the current is killed because of OOM */
 };
 
-static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
-                               int csize, bool oom_check)
+static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
+                               unsigned int nr_pages, bool oom_check)
 {
+       unsigned long csize = nr_pages * PAGE_SIZE;
        struct mem_cgroup *mem_over_limit;
        struct res_counter *fail_res;
        unsigned long flags = 0;
        int ret;
 
-       ret = res_counter_charge(&mem->res, csize, &fail_res);
+       ret = res_counter_charge(&memcg->res, csize, &fail_res);
 
        if (likely(!ret)) {
                if (!do_swap_account)
                        return CHARGE_OK;
-               ret = res_counter_charge(&mem->memsw, csize, &fail_res);
+               ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
                if (likely(!ret))
                        return CHARGE_OK;
 
-               res_counter_uncharge(&mem->res, csize);
+               res_counter_uncharge(&memcg->res, csize);
                mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
                flags |= MEM_CGROUP_RECLAIM_NOSWAP;
        } else
                mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
        /*
-        * csize can be either a huge page (HPAGE_SIZE), a batch of
-        * regular pages (CHARGE_SIZE), or a single regular page
-        * (PAGE_SIZE).
+        * nr_pages can be either a huge page (HPAGE_PMD_NR), a batch
+        * of regular pages (CHARGE_BATCH), or a single regular page (1).
         *
         * Never reclaim on behalf of optional batching, retry with a
         * single page instead.
         */
-       if (csize == CHARGE_SIZE)
+       if (nr_pages == CHARGE_BATCH)
                return CHARGE_RETRY;
 
        if (!(gfp_mask & __GFP_WAIT))
                return CHARGE_WOULDBLOCK;
 
-       ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
-                                             gfp_mask, flags);
-       if (mem_cgroup_margin(mem_over_limit) >= csize)
+       ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
+       if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
                return CHARGE_RETRY;
        /*
         * Even though the limit is exceeded at this point, reclaim
@@ -1887,7 +2164,7 @@ static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
         * unlikely to succeed so close to the limit, and we fall back
         * to regular pages anyway in case of failure.
         */
-       if (csize == PAGE_SIZE && ret)
+       if (nr_pages == 1 && ret)
                return CHARGE_RETRY;
 
        /*
@@ -1908,18 +2185,36 @@ static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
 }
 
 /*
- * Unlike exported interface, "oom" parameter is added. if oom==true,
- * oom-killer can be invoked.
+ * __mem_cgroup_try_charge() does
+ * 1. detect memcg to be charged against from passed *mm and *ptr,
+ * 2. update res_counter
+ * 3. call memory reclaim if necessary.
+ *
+ * In some special case, if the task is fatal, fatal_signal_pending() or
+ * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
+ * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
+ * as possible without any hazards. 2: all pages should have a valid
+ * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
+ * pointer, that is treated as a charge to root_mem_cgroup.
+ *
+ * So __mem_cgroup_try_charge() will return
+ *  0       ...  on success, filling *ptr with a valid memcg pointer.
+ *  -ENOMEM ...  charge failure because of resource limits.
+ *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
+ *
+ * Unlike the exported interface, an "oom" parameter is added. if oom==true,
+ * the oom-killer can be invoked.
  */
 static int __mem_cgroup_try_charge(struct mm_struct *mm,
                                   gfp_t gfp_mask,
-                                  struct mem_cgroup **memcg, bool oom,
-                                  int page_size)
+                                  unsigned int nr_pages,
+                                  struct mem_cgroup **ptr,
+                                  bool oom)
 {
+       unsigned int batch = max(CHARGE_BATCH, nr_pages);
        int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        int ret;
-       int csize = max(CHARGE_SIZE, (unsigned long) page_size);
 
        /*
         * Unlike gloval-vm's OOM-kill, we're not in memory shortage
@@ -1936,17 +2231,17 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
         * thread group leader migrates. It's possible that mm is not
         * set, if so charge the init_mm (happens for pagecache usage).
         */
-       if (!*memcg && !mm)
-               goto bypass;
+       if (!*ptr && !mm)
+               *ptr = root_mem_cgroup;
 again:
-       if (*memcg) { /* css should be a valid one */
-               mem = *memcg;
-               VM_BUG_ON(css_is_removed(&mem->css));
-               if (mem_cgroup_is_root(mem))
+       if (*ptr) { /* css should be a valid one */
+               memcg = *ptr;
+               VM_BUG_ON(css_is_removed(&memcg->css));
+               if (mem_cgroup_is_root(memcg))
                        goto done;
-               if (page_size == PAGE_SIZE && consume_stock(mem))
+               if (nr_pages == 1 && consume_stock(memcg))
                        goto done;
-               css_get(&mem->css);
+               css_get(&memcg->css);
        } else {
                struct task_struct *p;
 
@@ -1954,7 +2249,7 @@ again:
                p = rcu_dereference(mm->owner);
                /*
                 * Because we don't have task_lock(), "p" can exit.
-                * In that case, "mem" can point to root or p can be NULL with
+                * In that case, "memcg" can point to root or p can be NULL with
                 * race with swapoff. Then, we have small risk of mis-accouning.
                 * But such kind of mis-account by race always happens because
                 * we don't have cgroup_mutex(). It's overkill and we allo that
@@ -1962,12 +2257,14 @@ again:
                 * (*) swapoff at el will charge against mm-struct not against
                 * task-struct. So, mm->owner can be NULL.
                 */
-               mem = mem_cgroup_from_task(p);
-               if (!mem || mem_cgroup_is_root(mem)) {
+               memcg = mem_cgroup_from_task(p);
+               if (!memcg)
+                       memcg = root_mem_cgroup;
+               if (mem_cgroup_is_root(memcg)) {
                        rcu_read_unlock();
                        goto done;
                }
-               if (page_size == PAGE_SIZE && consume_stock(mem)) {
+               if (nr_pages == 1 && consume_stock(memcg)) {
                        /*
                         * It seems dagerous to access memcg without css_get().
                         * But considering how consume_stok works, it's not
@@ -1980,7 +2277,7 @@ again:
                        goto done;
                }
                /* after here, we may be blocked. we need to get refcnt */
-               if (!css_tryget(&mem->css)) {
+               if (!css_tryget(&memcg->css)) {
                        rcu_read_unlock();
                        goto again;
                }
@@ -1992,7 +2289,7 @@ again:
 
                /* If killed, bypass charge */
                if (fatal_signal_pending(current)) {
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto bypass;
                }
 
@@ -2002,45 +2299,44 @@ again:
                        nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
                }
 
-               ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
-
+               ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, oom_check);
                switch (ret) {
                case CHARGE_OK:
                        break;
                case CHARGE_RETRY: /* not in OOM situation but retry */
-                       csize = page_size;
-                       css_put(&mem->css);
-                       mem = NULL;
+                       batch = nr_pages;
+                       css_put(&memcg->css);
+                       memcg = NULL;
                        goto again;
                case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto nomem;
                case CHARGE_NOMEM: /* OOM routine works */
                        if (!oom) {
-                               css_put(&mem->css);
+                               css_put(&memcg->css);
                                goto nomem;
                        }
                        /* If oom, we never return -ENOMEM */
                        nr_oom_retries--;
                        break;
                case CHARGE_OOM_DIE: /* Killed by OOM Killer */
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto bypass;
                }
        } while (ret != CHARGE_OK);
 
-       if (csize > page_size)
-               refill_stock(mem, csize - page_size);
-       css_put(&mem->css);
+       if (batch > nr_pages)
+               refill_stock(memcg, batch - nr_pages);
+       css_put(&memcg->css);
 done:
-       *memcg = mem;
+       *ptr = memcg;
        return 0;
 nomem:
-       *memcg = NULL;
+       *ptr = NULL;
        return -ENOMEM;
 bypass:
-       *memcg = NULL;
-       return 0;
+       *ptr = root_mem_cgroup;
+       return -EINTR;
 }
 
 /*
@@ -2048,22 +2344,18 @@ bypass:
  * This function is for that and do uncharge, put css's refcnt.
  * gotten by try_charge().
  */
-static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
-                                                       unsigned long count)
+static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
+                                      unsigned int nr_pages)
 {
-       if (!mem_cgroup_is_root(mem)) {
-               res_counter_uncharge(&mem->res, PAGE_SIZE * count);
+       if (!mem_cgroup_is_root(memcg)) {
+               unsigned long bytes = nr_pages * PAGE_SIZE;
+
+               res_counter_uncharge(&memcg->res, bytes);
                if (do_swap_account)
-                       res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
+                       res_counter_uncharge(&memcg->memsw, bytes);
        }
 }
 
-static void mem_cgroup_cancel_charge(struct mem_cgroup *mem,
-                                    int page_size)
-{
-       __mem_cgroup_cancel_charge(mem, page_size >> PAGE_SHIFT);
-}
-
 /*
  * A helper function to get mem_cgroup from ID. must be called under
  * rcu_read_lock(). The caller must check css_is_removed() or some if
@@ -2085,7 +2377,7 @@ static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
 
 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        struct page_cgroup *pc;
        unsigned short id;
        swp_entry_t ent;
@@ -2095,40 +2387,39 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
        pc = lookup_page_cgroup(page);
        lock_page_cgroup(pc);
        if (PageCgroupUsed(pc)) {
-               mem = pc->mem_cgroup;
-               if (mem && !css_tryget(&mem->css))
-                       mem = NULL;
+               memcg = pc->mem_cgroup;
+               if (memcg && !css_tryget(&memcg->css))
+                       memcg = NULL;
        } else if (PageSwapCache(page)) {
                ent.val = page_private(page);
-               id = lookup_swap_cgroup(ent);
+               id = lookup_swap_cgroup_id(ent);
                rcu_read_lock();
-               mem = mem_cgroup_lookup(id);
-               if (mem && !css_tryget(&mem->css))
-                       mem = NULL;
+               memcg = mem_cgroup_lookup(id);
+               if (memcg && !css_tryget(&memcg->css))
+                       memcg = NULL;
                rcu_read_unlock();
        }
        unlock_page_cgroup(pc);
-       return mem;
+       return memcg;
 }
 
-static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
+static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
+                                      struct page *page,
+                                      unsigned int nr_pages,
                                       struct page_cgroup *pc,
-                                      enum charge_type ctype,
-                                      int page_size)
+                                      enum charge_type ctype)
 {
-       int nr_pages = page_size >> PAGE_SHIFT;
-
        lock_page_cgroup(pc);
        if (unlikely(PageCgroupUsed(pc))) {
                unlock_page_cgroup(pc);
-               mem_cgroup_cancel_charge(mem, page_size);
+               __mem_cgroup_cancel_charge(memcg, nr_pages);
                return;
        }
        /*
         * we don't need page_cgroup_lock about tail pages, becase they are not
         * accessed by any other context at this point.
         */
-       pc->mem_cgroup = mem;
+       pc->mem_cgroup = memcg;
        /*
         * We access a page_cgroup asynchronously without lock_page_cgroup().
         * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
@@ -2151,60 +2442,48 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
                break;
        }
 
-       mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), nr_pages);
        unlock_page_cgroup(pc);
+       WARN_ON_ONCE(PageLRU(page));
        /*
         * "charge_statistics" updated event counter. Then, check it.
         * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
         * if they exceeds softlimit.
         */
-       memcg_check_events(mem, pc->page);
+       memcg_check_events(memcg, page);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 
 #define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
-                       (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
+                       (1 << PCG_MIGRATION))
 /*
  * Because tail pages are not marked as "used", set it. We're under
- * zone->lru_lock, 'splitting on pmd' and compund_lock.
+ * zone->lru_lock, 'splitting on pmd' and compound_lock.
+ * charge/uncharge will be never happen and move_account() is done under
+ * compound_lock(), so we don't have to take care of races.
  */
-void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
+void mem_cgroup_split_huge_fixup(struct page *head)
 {
        struct page_cgroup *head_pc = lookup_page_cgroup(head);
-       struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
-       unsigned long flags;
+       struct page_cgroup *pc;
+       int i;
 
        if (mem_cgroup_disabled())
                return;
-       /*
-        * We have no races with charge/uncharge but will have races with
-        * page state accounting.
-        */
-       move_lock_page_cgroup(head_pc, &flags);
-
-       tail_pc->mem_cgroup = head_pc->mem_cgroup;
-       smp_wmb(); /* see __commit_charge() */
-       if (PageCgroupAcctLRU(head_pc)) {
-               enum lru_list lru;
-               struct mem_cgroup_per_zone *mz;
-
-               /*
-                * LRU flags cannot be copied because we need to add tail
-                *.page to LRU by generic call and our hook will be called.
-                * We hold lru_lock, then, reduce counter directly.
-                */
-               lru = page_lru(head);
-               mz = page_cgroup_zoneinfo(head_pc);
-               MEM_CGROUP_ZSTAT(mz, lru) -= 1;
+       for (i = 1; i < HPAGE_PMD_NR; i++) {
+               pc = head_pc + i;
+               pc->mem_cgroup = head_pc->mem_cgroup;
+               smp_wmb();/* see __commit_charge() */
+               pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
        }
-       tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
-       move_unlock_page_cgroup(head_pc, &flags);
 }
-#endif
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 /**
- * __mem_cgroup_move_account - move account of the page
+ * mem_cgroup_move_account - move account of the page
+ * @page: the page
+ * @nr_pages: number of regular pages (>1 for huge pages)
  * @pc:        page_cgroup of the page.
  * @from: mem_cgroup which the page is moved from.
  * @to:        mem_cgroup which the page is moved to. @from != @to.
@@ -2212,25 +2491,42 @@ void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
  *
  * The caller must confirm following.
  * - page is not on LRU (isolate_page() is useful.)
- * - the pc is locked, used, and ->mem_cgroup points to @from.
+ * - compound_lock is held when nr_pages > 1
  *
  * This function doesn't do "charge" nor css_get to new cgroup. It should be
- * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
+ * done by a caller(__mem_cgroup_try_charge would be useful). If @uncharge is
  * true, this function does "uncharge" from old cgroup, but it doesn't if
  * @uncharge is false, so a caller should do "uncharge".
  */
-
-static void __mem_cgroup_move_account(struct page_cgroup *pc,
-       struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge,
-       int charge_size)
+static int mem_cgroup_move_account(struct page *page,
+                                  unsigned int nr_pages,
+                                  struct page_cgroup *pc,
+                                  struct mem_cgroup *from,
+                                  struct mem_cgroup *to,
+                                  bool uncharge)
 {
-       int nr_pages = charge_size >> PAGE_SHIFT;
+       unsigned long flags;
+       int ret;
 
        VM_BUG_ON(from == to);
-       VM_BUG_ON(PageLRU(pc->page));
-       VM_BUG_ON(!page_is_cgroup_locked(pc));
-       VM_BUG_ON(!PageCgroupUsed(pc));
-       VM_BUG_ON(pc->mem_cgroup != from);
+       VM_BUG_ON(PageLRU(page));
+       /*
+        * The page is isolated from LRU. So, collapse function
+        * will not handle this page. But page splitting can happen.
+        * Do this check under compound_page_lock(). The caller should
+        * hold it.
+        */
+       ret = -EBUSY;
+       if (nr_pages > 1 && !PageTransHuge(page))
+               goto out;
+
+       lock_page_cgroup(pc);
+
+       ret = -EINVAL;
+       if (!PageCgroupUsed(pc) || pc->mem_cgroup != from)
+               goto unlock;
+
+       move_lock_page_cgroup(pc, &flags);
 
        if (PageCgroupFileMapped(pc)) {
                /* Update mapped_file data for mem_cgroup */
@@ -2242,7 +2538,7 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
        mem_cgroup_charge_statistics(from, PageCgroupCache(pc), -nr_pages);
        if (uncharge)
                /* This is not "cancel", but cancel_charge does all we need. */
-               mem_cgroup_cancel_charge(from, charge_size);
+               __mem_cgroup_cancel_charge(from, nr_pages);
 
        /* caller should have done css_get */
        pc->mem_cgroup = to;
@@ -2251,43 +2547,19 @@ static void __mem_cgroup_move_account(struct page_cgroup *pc,
         * We charges against "to" which may not have any tasks. Then, "to"
         * can be under rmdir(). But in current implementation, caller of
         * this function is just force_empty() and move charge, so it's
-        * garanteed that "to" is never removed. So, we don't check rmdir
+        * guaranteed that "to" is never removed. So, we don't check rmdir
         * status here.
         */
-}
-
-/*
- * check whether the @pc is valid for moving account and call
- * __mem_cgroup_move_account()
- */
-static int mem_cgroup_move_account(struct page_cgroup *pc,
-               struct mem_cgroup *from, struct mem_cgroup *to,
-               bool uncharge, int charge_size)
-{
-       int ret = -EINVAL;
-       unsigned long flags;
-       /*
-        * The page is isolated from LRU. So, collapse function
-        * will not handle this page. But page splitting can happen.
-        * Do this check under compound_page_lock(). The caller should
-        * hold it.
-        */
-       if ((charge_size > PAGE_SIZE) && !PageTransHuge(pc->page))
-               return -EBUSY;
-
-       lock_page_cgroup(pc);
-       if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
-               move_lock_page_cgroup(pc, &flags);
-               __mem_cgroup_move_account(pc, from, to, uncharge, charge_size);
-               move_unlock_page_cgroup(pc, &flags);
-               ret = 0;
-       }
+       move_unlock_page_cgroup(pc, &flags);
+       ret = 0;
+unlock:
        unlock_page_cgroup(pc);
        /*
         * check events
         */
-       memcg_check_events(to, pc->page);
-       memcg_check_events(from, pc->page);
+       memcg_check_events(to, page);
+       memcg_check_events(from, page);
+out:
        return ret;
 }
 
@@ -2295,16 +2567,16 @@ static int mem_cgroup_move_account(struct page_cgroup *pc,
  * move charges to its parent.
  */
 
-static int mem_cgroup_move_parent(struct page_cgroup *pc,
+static int mem_cgroup_move_parent(struct page *page,
+                                 struct page_cgroup *pc,
                                  struct mem_cgroup *child,
                                  gfp_t gfp_mask)
 {
-       struct page *page = pc->page;
        struct cgroup *cg = child->css.cgroup;
        struct cgroup *pcg = cg->parent;
        struct mem_cgroup *parent;
-       int page_size = PAGE_SIZE;
-       unsigned long flags;
+       unsigned int nr_pages;
+       unsigned long uninitialized_var(flags);
        int ret;
 
        /* Is ROOT ? */
@@ -2317,23 +2589,21 @@ static int mem_cgroup_move_parent(struct page_cgroup *pc,
        if (isolate_lru_page(page))
                goto put;
 
-       if (PageTransHuge(page))
-               page_size = HPAGE_SIZE;
+       nr_pages = hpage_nr_pages(page);
 
        parent = mem_cgroup_from_cont(pcg);
-       ret = __mem_cgroup_try_charge(NULL, gfp_mask,
-                               &parent, false, page_size);
-       if (ret || !parent)
+       ret = __mem_cgroup_try_charge(NULL, gfp_mask, nr_pages, &parent, false);
+       if (ret)
                goto put_back;
 
-       if (page_size > PAGE_SIZE)
+       if (nr_pages > 1)
                flags = compound_lock_irqsave(page);
 
-       ret = mem_cgroup_move_account(pc, child, parent, true, page_size);
+       ret = mem_cgroup_move_account(page, nr_pages, pc, child, parent, true);
        if (ret)
-               mem_cgroup_cancel_charge(parent, page_size);
+               __mem_cgroup_cancel_charge(parent, nr_pages);
 
-       if (page_size > PAGE_SIZE)
+       if (nr_pages > 1)
                compound_unlock_irqrestore(page, flags);
 put_back:
        putback_lru_page(page);
@@ -2352,14 +2622,14 @@ out:
 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
                                gfp_t gfp_mask, enum charge_type ctype)
 {
-       struct mem_cgroup *mem = NULL;
-       int page_size = PAGE_SIZE;
+       struct mem_cgroup *memcg = NULL;
+       unsigned int nr_pages = 1;
        struct page_cgroup *pc;
        bool oom = true;
        int ret;
 
        if (PageTransHuge(page)) {
-               page_size <<= compound_order(page);
+               nr_pages <<= compound_order(page);
                VM_BUG_ON(!PageTransHuge(page));
                /*
                 * Never OOM-kill a process for a huge page.  The
@@ -2369,13 +2639,10 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
        }
 
        pc = lookup_page_cgroup(page);
-       BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */
-
-       ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, oom, page_size);
-       if (ret || !mem)
+       ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
+       if (ret == -ENOMEM)
                return ret;
-
-       __mem_cgroup_commit_charge(mem, pc, ctype, page_size);
+       __mem_cgroup_commit_charge(memcg, page, nr_pages, pc, ctype);
        return 0;
 }
 
@@ -2384,78 +2651,70 @@ int mem_cgroup_newpage_charge(struct page *page,
 {
        if (mem_cgroup_disabled())
                return 0;
-       /*
-        * If already mapped, we don't have to account.
-        * If page cache, page->mapping has address_space.
-        * But page->mapping may have out-of-use anon_vma pointer,
-        * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
-        * is NULL.
-        */
-       if (page_mapped(page) || (page->mapping && !PageAnon(page)))
-               return 0;
-       if (unlikely(!mm))
-               mm = &init_mm;
+       VM_BUG_ON(page_mapped(page));
+       VM_BUG_ON(page->mapping && !PageAnon(page));
+       VM_BUG_ON(!mm);
        return mem_cgroup_charge_common(page, mm, gfp_mask,
-                               MEM_CGROUP_CHARGE_TYPE_MAPPED);
+                                       MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
 static void
 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
                                        enum charge_type ctype);
 
+static void
+__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
+                                       enum charge_type ctype)
+{
+       struct page_cgroup *pc = lookup_page_cgroup(page);
+       struct zone *zone = page_zone(page);
+       unsigned long flags;
+       bool removed = false;
+
+       /*
+        * In some case, SwapCache, FUSE(splice_buf->radixtree), the page
+        * is already on LRU. It means the page may on some other page_cgroup's
+        * LRU. Take care of it.
+        */
+       spin_lock_irqsave(&zone->lru_lock, flags);
+       if (PageLRU(page)) {
+               del_page_from_lru_list(zone, page, page_lru(page));
+               ClearPageLRU(page);
+               removed = true;
+       }
+       __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
+       if (removed) {
+               add_page_to_lru_list(zone, page, page_lru(page));
+               SetPageLRU(page);
+       }
+       spin_unlock_irqrestore(&zone->lru_lock, flags);
+       return;
+}
+
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                gfp_t gfp_mask)
 {
+       struct mem_cgroup *memcg = NULL;
+       enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
        int ret;
 
        if (mem_cgroup_disabled())
                return 0;
        if (PageCompound(page))
                return 0;
-       /*
-        * Corner case handling. This is called from add_to_page_cache()
-        * in usual. But some FS (shmem) precharges this page before calling it
-        * and call add_to_page_cache() with GFP_NOWAIT.
-        *
-        * For GFP_NOWAIT case, the page may be pre-charged before calling
-        * add_to_page_cache(). (See shmem.c) check it here and avoid to call
-        * charge twice. (It works but has to pay a bit larger cost.)
-        * And when the page is SwapCache, it should take swap information
-        * into account. This is under lock_page() now.
-        */
-       if (!(gfp_mask & __GFP_WAIT)) {
-               struct page_cgroup *pc;
-
-               pc = lookup_page_cgroup(page);
-               if (!pc)
-                       return 0;
-               lock_page_cgroup(pc);
-               if (PageCgroupUsed(pc)) {
-                       unlock_page_cgroup(pc);
-                       return 0;
-               }
-               unlock_page_cgroup(pc);
-       }
 
        if (unlikely(!mm))
                mm = &init_mm;
+       if (!page_is_file_cache(page))
+               type = MEM_CGROUP_CHARGE_TYPE_SHMEM;
 
-       if (page_is_file_cache(page))
-               return mem_cgroup_charge_common(page, mm, gfp_mask,
-                               MEM_CGROUP_CHARGE_TYPE_CACHE);
-
-       /* shmem */
-       if (PageSwapCache(page)) {
-               struct mem_cgroup *mem;
-
-               ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
+       if (!PageSwapCache(page))
+               ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
+       else { /* page is swapcache/shmem */
+               ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &memcg);
                if (!ret)
-                       __mem_cgroup_commit_charge_swapin(page, mem,
-                                       MEM_CGROUP_CHARGE_TYPE_SHMEM);
-       } else
-               ret = mem_cgroup_charge_common(page, mm, gfp_mask,
-                                       MEM_CGROUP_CHARGE_TYPE_SHMEM);
-
+                       __mem_cgroup_commit_charge_swapin(page, memcg, type);
+       }
        return ret;
 }
 
@@ -2467,12 +2726,12 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  */
 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
                                 struct page *page,
-                                gfp_t mask, struct mem_cgroup **ptr)
+                                gfp_t mask, struct mem_cgroup **memcgp)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        int ret;
 
-       *ptr = NULL;
+       *memcgp = NULL;
 
        if (mem_cgroup_disabled())
                return 0;
@@ -2487,34 +2746,35 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
         */
        if (!PageSwapCache(page))
                goto charge_cur_mm;
-       mem = try_get_mem_cgroup_from_page(page);
-       if (!mem)
+       memcg = try_get_mem_cgroup_from_page(page);
+       if (!memcg)
                goto charge_cur_mm;
-       *ptr = mem;
-       ret = __mem_cgroup_try_charge(NULL, mask, ptr, true, PAGE_SIZE);
-       css_put(&mem->css);
+       *memcgp = memcg;
+       ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
+       css_put(&memcg->css);
+       if (ret == -EINTR)
+               ret = 0;
        return ret;
 charge_cur_mm:
        if (unlikely(!mm))
                mm = &init_mm;
-       return __mem_cgroup_try_charge(mm, mask, ptr, true, PAGE_SIZE);
+       ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
+       if (ret == -EINTR)
+               ret = 0;
+       return ret;
 }
 
 static void
-__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
+__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
                                        enum charge_type ctype)
 {
-       struct page_cgroup *pc;
-
        if (mem_cgroup_disabled())
                return;
-       if (!ptr)
+       if (!memcg)
                return;
-       cgroup_exclude_rmdir(&ptr->css);
-       pc = lookup_page_cgroup(page);
-       mem_cgroup_lru_del_before_commit_swapcache(page);
-       __mem_cgroup_commit_charge(ptr, pc, ctype, PAGE_SIZE);
-       mem_cgroup_lru_add_after_commit_swapcache(page);
+       cgroup_exclude_rmdir(&memcg->css);
+
+       __mem_cgroup_commit_charge_lrucare(page, memcg, ctype);
        /*
         * Now swap is on-memory. This means this page may be
         * counted both as mem and swap....double count.
@@ -2524,21 +2784,22 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
         */
        if (do_swap_account && PageSwapCache(page)) {
                swp_entry_t ent = {.val = page_private(page)};
+               struct mem_cgroup *swap_memcg;
                unsigned short id;
-               struct mem_cgroup *memcg;
 
                id = swap_cgroup_record(ent, 0);
                rcu_read_lock();
-               memcg = mem_cgroup_lookup(id);
-               if (memcg) {
+               swap_memcg = mem_cgroup_lookup(id);
+               if (swap_memcg) {
                        /*
                         * This recorded memcg can be obsolete one. So, avoid
                         * calling css_tryget
                         */
-                       if (!mem_cgroup_is_root(memcg))
-                               res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
-                       mem_cgroup_swap_statistics(memcg, false);
-                       mem_cgroup_put(memcg);
+                       if (!mem_cgroup_is_root(swap_memcg))
+                               res_counter_uncharge(&swap_memcg->memsw,
+                                                    PAGE_SIZE);
+                       mem_cgroup_swap_statistics(swap_memcg, false);
+                       mem_cgroup_put(swap_memcg);
                }
                rcu_read_unlock();
        }
@@ -2547,30 +2808,32 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
         * So, rmdir()->pre_destroy() can be called while we do this charge.
         * In that case, we need to call pre_destroy() again. check it here.
         */
-       cgroup_release_and_wakeup_rmdir(&ptr->css);
+       cgroup_release_and_wakeup_rmdir(&memcg->css);
 }
 
-void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
+void mem_cgroup_commit_charge_swapin(struct page *page,
+                                    struct mem_cgroup *memcg)
 {
-       __mem_cgroup_commit_charge_swapin(page, ptr,
-                                       MEM_CGROUP_CHARGE_TYPE_MAPPED);
+       __mem_cgroup_commit_charge_swapin(page, memcg,
+                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
-void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
+void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
 {
        if (mem_cgroup_disabled())
                return;
-       if (!mem)
+       if (!memcg)
                return;
-       mem_cgroup_cancel_charge(mem, PAGE_SIZE);
+       __mem_cgroup_cancel_charge(memcg, 1);
 }
 
-static void
-__do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
-             int page_size)
+static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
+                                  unsigned int nr_pages,
+                                  const enum charge_type ctype)
 {
        struct memcg_batch_info *batch = NULL;
        bool uncharge_memsw = true;
+
        /* If swapout, usage of swap doesn't decrease */
        if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
                uncharge_memsw = false;
@@ -2582,10 +2845,10 @@ __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
         * uncharges. Then, it's ok to ignore memcg's refcnt.
         */
        if (!batch->memcg)
-               batch->memcg = mem;
+               batch->memcg = memcg;
        /*
         * do_batch > 0 when unmapping pages or inode invalidate/truncate.
-        * In those cases, all pages freed continously can be expected to be in
+        * In those cases, all pages freed continuously can be expected to be in
         * the same cgroup and we have chance to coalesce uncharges.
         * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
         * because we want to do uncharge as soon as possible.
@@ -2594,7 +2857,7 @@ __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
        if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
                goto direct_uncharge;
 
-       if (page_size != PAGE_SIZE)
+       if (nr_pages > 1)
                goto direct_uncharge;
 
        /*
@@ -2602,19 +2865,19 @@ __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype,
         * merge a series of uncharges to an uncharge of res_counter.
         * If not, we uncharge res_counter ony by one.
         */
-       if (batch->memcg != mem)
+       if (batch->memcg != memcg)
                goto direct_uncharge;
        /* remember freed charge and uncharge it later */
-       batch->bytes += PAGE_SIZE;
+       batch->nr_pages++;
        if (uncharge_memsw)
-               batch->memsw_bytes += PAGE_SIZE;
+               batch->memsw_nr_pages++;
        return;
 direct_uncharge:
-       res_counter_uncharge(&mem->res, page_size);
+       res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
        if (uncharge_memsw)
-               res_counter_uncharge(&mem->memsw, page_size);
-       if (unlikely(batch->memcg != mem))
-               memcg_oom_recover(mem);
+               res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
+       if (unlikely(batch->memcg != memcg))
+               memcg_oom_recover(memcg);
        return;
 }
 
@@ -2624,10 +2887,9 @@ direct_uncharge:
 static struct mem_cgroup *
 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 {
-       int count;
+       struct mem_cgroup *memcg = NULL;
+       unsigned int nr_pages = 1;
        struct page_cgroup *pc;
-       struct mem_cgroup *mem = NULL;
-       int page_size = PAGE_SIZE;
 
        if (mem_cgroup_disabled())
                return NULL;
@@ -2636,21 +2898,19 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                return NULL;
 
        if (PageTransHuge(page)) {
-               page_size <<= compound_order(page);
+               nr_pages <<= compound_order(page);
                VM_BUG_ON(!PageTransHuge(page));
        }
-
-       count = page_size >> PAGE_SHIFT;
        /*
         * Check if our page_cgroup is valid
         */
        pc = lookup_page_cgroup(page);
-       if (unlikely(!pc || !PageCgroupUsed(pc)))
+       if (unlikely(!PageCgroupUsed(pc)))
                return NULL;
 
        lock_page_cgroup(pc);
 
-       mem = pc->mem_cgroup;
+       memcg = pc->mem_cgroup;
 
        if (!PageCgroupUsed(pc))
                goto unlock_out;
@@ -2673,7 +2933,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                break;
        }
 
-       mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -count);
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -nr_pages);
 
        ClearPageCgroupUsed(pc);
        /*
@@ -2685,18 +2945,18 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 
        unlock_page_cgroup(pc);
        /*
-        * even after unlock, we have mem->res.usage here and this memcg
+        * even after unlock, we have memcg->res.usage here and this memcg
         * will never be freed.
         */
-       memcg_check_events(mem, page);
+       memcg_check_events(memcg, page);
        if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
-               mem_cgroup_swap_statistics(mem, true);
-               mem_cgroup_get(mem);
+               mem_cgroup_swap_statistics(memcg, true);
+               mem_cgroup_get(memcg);
        }
-       if (!mem_cgroup_is_root(mem))
-               __do_uncharge(mem, ctype, page_size);
+       if (!mem_cgroup_is_root(memcg))
+               mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
 
-       return mem;
+       return memcg;
 
 unlock_out:
        unlock_page_cgroup(pc);
@@ -2708,8 +2968,7 @@ void mem_cgroup_uncharge_page(struct page *page)
        /* early check. */
        if (page_mapped(page))
                return;
-       if (page->mapping && !PageAnon(page))
-               return;
+       VM_BUG_ON(page->mapping && !PageAnon(page));
        __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
@@ -2734,8 +2993,8 @@ void mem_cgroup_uncharge_start(void)
        /* We can do nest. */
        if (current->memcg_batch.do_batch == 1) {
                current->memcg_batch.memcg = NULL;
-               current->memcg_batch.bytes = 0;
-               current->memcg_batch.memsw_bytes = 0;
+               current->memcg_batch.nr_pages = 0;
+               current->memcg_batch.memsw_nr_pages = 0;
        }
 }
 
@@ -2756,15 +3015,34 @@ void mem_cgroup_uncharge_end(void)
         * This "batch->memcg" is valid without any css_get/put etc...
         * bacause we hide charges behind us.
         */
-       if (batch->bytes)
-               res_counter_uncharge(&batch->memcg->res, batch->bytes);
-       if (batch->memsw_bytes)
-               res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
+       if (batch->nr_pages)
+               res_counter_uncharge(&batch->memcg->res,
+                                    batch->nr_pages * PAGE_SIZE);
+       if (batch->memsw_nr_pages)
+               res_counter_uncharge(&batch->memcg->memsw,
+                                    batch->memsw_nr_pages * PAGE_SIZE);
        memcg_oom_recover(batch->memcg);
        /* forget this pointer (for sanity check) */
        batch->memcg = NULL;
 }
 
+/*
+ * A function for resetting pc->mem_cgroup for newly allocated pages.
+ * This function should be called if the newpage will be added to LRU
+ * before start accounting.
+ */
+void mem_cgroup_reset_owner(struct page *newpage)
+{
+       struct page_cgroup *pc;
+
+       if (mem_cgroup_disabled())
+               return;
+
+       pc = lookup_page_cgroup(newpage);
+       VM_BUG_ON(PageCgroupUsed(pc));
+       pc->mem_cgroup = root_mem_cgroup;
+}
+
 #ifdef CONFIG_SWAP
 /*
  * called after __delete_from_swap_cache() and drop "page" account.
@@ -2882,14 +3160,14 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  * page belongs to.
  */
 int mem_cgroup_prepare_migration(struct page *page,
-       struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
+       struct page *newpage, struct mem_cgroup **memcgp, gfp_t gfp_mask)
 {
+       struct mem_cgroup *memcg = NULL;
        struct page_cgroup *pc;
-       struct mem_cgroup *mem = NULL;
        enum charge_type ctype;
        int ret = 0;
 
-       *ptr = NULL;
+       *memcgp = NULL;
 
        VM_BUG_ON(PageTransHuge(page));
        if (mem_cgroup_disabled())
@@ -2898,8 +3176,8 @@ int mem_cgroup_prepare_migration(struct page *page,
        pc = lookup_page_cgroup(page);
        lock_page_cgroup(pc);
        if (PageCgroupUsed(pc)) {
-               mem = pc->mem_cgroup;
-               css_get(&mem->css);
+               memcg = pc->mem_cgroup;
+               css_get(&memcg->css);
                /*
                 * At migrating an anonymous page, its mapcount goes down
                 * to 0 and uncharge() will be called. But, even if it's fully
@@ -2937,13 +3215,13 @@ int mem_cgroup_prepare_migration(struct page *page,
         * If the page is not charged at this point,
         * we return here.
         */
-       if (!mem)
+       if (!memcg)
                return 0;
 
-       *ptr = mem;
-       ret = __mem_cgroup_try_charge(NULL, gfp_mask, ptr, false, PAGE_SIZE);
-       css_put(&mem->css);/* drop extra refcnt */
-       if (ret || *ptr == NULL) {
+       *memcgp = memcg;
+       ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, memcgp, false);
+       css_put(&memcg->css);/* drop extra refcnt */
+       if (ret) {
                if (PageAnon(page)) {
                        lock_page_cgroup(pc);
                        ClearPageCgroupMigration(pc);
@@ -2953,6 +3231,7 @@ int mem_cgroup_prepare_migration(struct page *page,
                         */
                        mem_cgroup_uncharge_page(page);
                }
+               /* we'll need to revisit this error code (we have -EINTR) */
                return -ENOMEM;
        }
        /*
@@ -2968,21 +3247,21 @@ int mem_cgroup_prepare_migration(struct page *page,
                ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
        else
                ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
-       __mem_cgroup_commit_charge(mem, pc, ctype, PAGE_SIZE);
+       __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
        return ret;
 }
 
 /* remove redundant charge if migration failed*/
-void mem_cgroup_end_migration(struct mem_cgroup *mem,
+void mem_cgroup_end_migration(struct mem_cgroup *memcg,
        struct page *oldpage, struct page *newpage, bool migration_ok)
 {
        struct page *used, *unused;
        struct page_cgroup *pc;
 
-       if (!mem)
+       if (!memcg)
                return;
        /* blocks rmdir() */
-       cgroup_exclude_rmdir(&mem->css);
+       cgroup_exclude_rmdir(&memcg->css);
        if (!migration_ok) {
                used = oldpage;
                unused = newpage;
@@ -3018,32 +3297,41 @@ void mem_cgroup_end_migration(struct mem_cgroup *mem,
         * So, rmdir()->pre_destroy() can be called while we do this charge.
         * In that case, we need to call pre_destroy() again. check it here.
         */
-       cgroup_release_and_wakeup_rmdir(&mem->css);
+       cgroup_release_and_wakeup_rmdir(&memcg->css);
 }
 
 /*
- * A call to try to shrink memory usage on charge failure at shmem's swapin.
- * Calling hierarchical_reclaim is not enough because we should update
- * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
- * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
- * not from the memcg which this page would be charged to.
- * try_charge_swapin does all of these works properly.
+ * At replace page cache, newpage is not under any memcg but it's on
+ * LRU. So, this function doesn't touch res_counter but handles LRU
+ * in correct way. Both pages are locked so we cannot race with uncharge.
  */
-int mem_cgroup_shmem_charge_fallback(struct page *page,
-                           struct mm_struct *mm,
-                           gfp_t gfp_mask)
+void mem_cgroup_replace_page_cache(struct page *oldpage,
+                                 struct page *newpage)
 {
-       struct mem_cgroup *mem;
-       int ret;
+       struct mem_cgroup *memcg;
+       struct page_cgroup *pc;
+       enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
 
        if (mem_cgroup_disabled())
-               return 0;
+               return;
 
-       ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
-       if (!ret)
-               mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
+       pc = lookup_page_cgroup(oldpage);
+       /* fix accounting on old pages */
+       lock_page_cgroup(pc);
+       memcg = pc->mem_cgroup;
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -1);
+       ClearPageCgroupUsed(pc);
+       unlock_page_cgroup(pc);
 
-       return ret;
+       if (PageSwapBacked(oldpage))
+               type = MEM_CGROUP_CHARGE_TYPE_SHMEM;
+
+       /*
+        * Even if newpage->mapping was NULL before starting replacement,
+        * the newpage may be on LRU(or pagevec for LRU) already. We lock
+        * LRU while we overwrite pc->mem_cgroup.
+        */
+       __mem_cgroup_commit_charge_lrucare(newpage, memcg, type);
 }
 
 #ifdef CONFIG_DEBUG_VM
@@ -3052,6 +3340,11 @@ static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
        struct page_cgroup *pc;
 
        pc = lookup_page_cgroup(page);
+       /*
+        * Can be NULL while feeding pages into the page allocator for
+        * the first time, i.e. during boot or memory hotplug;
+        * or when mem_cgroup_disabled().
+        */
        if (likely(pc) && PageCgroupUsed(pc))
                return pc;
        return NULL;
@@ -3071,23 +3364,8 @@ void mem_cgroup_print_bad_page(struct page *page)
 
        pc = lookup_page_cgroup_used(page);
        if (pc) {
-               int ret = -1;
-               char *path;
-
-               printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p",
+               printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
                       pc, pc->flags, pc->mem_cgroup);
-
-               path = kmalloc(PATH_MAX, GFP_KERNEL);
-               if (path) {
-                       rcu_read_lock();
-                       ret = cgroup_path(pc->mem_cgroup->css.cgroup,
-                                                       path, PATH_MAX);
-                       rcu_read_unlock();
-               }
-
-               printk(KERN_CONT "(%s)\n",
-                               (ret < 0) ? "cannot get the path" : path);
-               kfree(path);
        }
 }
 #endif
@@ -3122,7 +3400,7 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                /*
                 * Rather than hide all in some function, I do this in
                 * open coded manner. You see what this really does.
-                * We have to guarantee mem->res.limit < mem->memsw.limit.
+                * We have to guarantee memcg->res.limit < memcg->memsw.limit.
                 */
                mutex_lock(&set_limit_mutex);
                memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
@@ -3148,8 +3426,8 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                if (!ret)
                        break;
 
-               mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
-                                               MEM_CGROUP_RECLAIM_SHRINK);
+               mem_cgroup_reclaim(memcg, GFP_KERNEL,
+                                  MEM_CGROUP_RECLAIM_SHRINK);
                curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
                /* Usage is reduced ? */
                if (curusage >= oldusage)
@@ -3183,7 +3461,7 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
                /*
                 * Rather than hide all in some function, I do this in
                 * open coded manner. You see what this really does.
-                * We have to guarantee mem->res.limit < mem->memsw.limit.
+                * We have to guarantee memcg->res.limit < memcg->memsw.limit.
                 */
                mutex_lock(&set_limit_mutex);
                memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
@@ -3207,9 +3485,9 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
                if (!ret)
                        break;
 
-               mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
-                                               MEM_CGROUP_RECLAIM_NOSWAP |
-                                               MEM_CGROUP_RECLAIM_SHRINK);
+               mem_cgroup_reclaim(memcg, GFP_KERNEL,
+                                  MEM_CGROUP_RECLAIM_NOSWAP |
+                                  MEM_CGROUP_RECLAIM_SHRINK);
                curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
                /* Usage is reduced ? */
                if (curusage >= oldusage)
@@ -3223,7 +3501,8 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
 }
 
 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
-                                           gfp_t gfp_mask)
+                                           gfp_t gfp_mask,
+                                           unsigned long *total_scanned)
 {
        unsigned long nr_reclaimed = 0;
        struct mem_cgroup_per_zone *mz, *next_mz = NULL;
@@ -3231,6 +3510,7 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
        int loop = 0;
        struct mem_cgroup_tree_per_zone *mctz;
        unsigned long long excess;
+       unsigned long nr_scanned;
 
        if (order > 0)
                return 0;
@@ -3249,10 +3529,11 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
                if (!mz)
                        break;
 
-               reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
-                                               gfp_mask,
-                                               MEM_CGROUP_RECLAIM_SOFT);
+               nr_scanned = 0;
+               reclaimed = mem_cgroup_soft_reclaim(mz->mem, zone,
+                                                   gfp_mask, &nr_scanned);
                nr_reclaimed += reclaimed;
+               *total_scanned += nr_scanned;
                spin_lock(&mctz->lock);
 
                /*
@@ -3275,10 +3556,9 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
                                 */
                                next_mz =
                                __mem_cgroup_largest_soft_limit_node(mctz);
-                               if (next_mz == mz) {
+                               if (next_mz == mz)
                                        css_put(&next_mz->mem->css);
-                                       next_mz = NULL;
-                               } else /* next_mz == NULL or other memcg */
+                               else /* next_mz == NULL or other memcg */
                                        break;
                        } while (1);
                }
@@ -3316,47 +3596,52 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  * This routine traverse page_cgroup in given list and drop them all.
  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  */
-static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
+static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
                                int node, int zid, enum lru_list lru)
 {
-       struct zone *zone;
        struct mem_cgroup_per_zone *mz;
-       struct page_cgroup *pc, *busy;
        unsigned long flags, loop;
        struct list_head *list;
+       struct page *busy;
+       struct zone *zone;
        int ret = 0;
 
        zone = &NODE_DATA(node)->node_zones[zid];
-       mz = mem_cgroup_zoneinfo(mem, node, zid);
-       list = &mz->lists[lru];
+       mz = mem_cgroup_zoneinfo(memcg, node, zid);
+       list = &mz->lruvec.lists[lru];
 
        loop = MEM_CGROUP_ZSTAT(mz, lru);
        /* give some margin against EBUSY etc...*/
        loop += 256;
        busy = NULL;
        while (loop--) {
+               struct page_cgroup *pc;
+               struct page *page;
+
                ret = 0;
                spin_lock_irqsave(&zone->lru_lock, flags);
                if (list_empty(list)) {
                        spin_unlock_irqrestore(&zone->lru_lock, flags);
                        break;
                }
-               pc = list_entry(list->prev, struct page_cgroup, lru);
-               if (busy == pc) {
-                       list_move(&pc->lru, list);
+               page = list_entry(list->prev, struct page, lru);
+               if (busy == page) {
+                       list_move(&page->lru, list);
                        busy = NULL;
                        spin_unlock_irqrestore(&zone->lru_lock, flags);
                        continue;
                }
                spin_unlock_irqrestore(&zone->lru_lock, flags);
 
-               ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
-               if (ret == -ENOMEM)
+               pc = lookup_page_cgroup(page);
+
+               ret = mem_cgroup_move_parent(page, pc, memcg, GFP_KERNEL);
+               if (ret == -ENOMEM || ret == -EINTR)
                        break;
 
                if (ret == -EBUSY || ret == -EINVAL) {
                        /* found lock contention or "pc" is obsolete. */
-                       busy = pc;
+                       busy = page;
                        cond_resched();
                } else
                        busy = NULL;
@@ -3371,14 +3656,14 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
  * make mem_cgroup's charge to be 0 if there is no task.
  * This enables deleting this mem_cgroup.
  */
-static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
+static int mem_cgroup_force_empty(struct mem_cgroup *memcg, bool free_all)
 {
        int ret;
        int node, zid, shrink;
        int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
-       struct cgroup *cgrp = mem->css.cgroup;
+       struct cgroup *cgrp = memcg->css.cgroup;
 
-       css_get(&mem->css);
+       css_get(&memcg->css);
 
        shrink = 0;
        /* should free all ? */
@@ -3394,14 +3679,14 @@ move_account:
                        goto out;
                /* This is for making all *used* pages to be on LRU. */
                lru_add_drain_all();
-               drain_all_stock_sync();
+               drain_all_stock_sync(memcg);
                ret = 0;
-               mem_cgroup_start_move(mem);
+               mem_cgroup_start_move(memcg);
                for_each_node_state(node, N_HIGH_MEMORY) {
                        for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
                                enum lru_list l;
                                for_each_lru(l) {
-                                       ret = mem_cgroup_force_empty_list(mem,
+                                       ret = mem_cgroup_force_empty_list(memcg,
                                                        node, zid, l);
                                        if (ret)
                                                break;
@@ -3410,16 +3695,16 @@ move_account:
                        if (ret)
                                break;
                }
-               mem_cgroup_end_move(mem);
-               memcg_oom_recover(mem);
+               mem_cgroup_end_move(memcg);
+               memcg_oom_recover(memcg);
                /* it seems parent cgroup doesn't have enough mem */
                if (ret == -ENOMEM)
                        goto try_to_free;
                cond_resched();
        /* "ret" should also be checked to ensure all lists are empty. */
-       } while (mem->res.usage > 0 || ret);
+       } while (memcg->res.usage > 0 || ret);
 out:
-       css_put(&mem->css);
+       css_put(&memcg->css);
        return ret;
 
 try_to_free:
@@ -3432,15 +3717,15 @@ try_to_free:
        lru_add_drain_all();
        /* try to free all pages in this cgroup */
        shrink = 1;
-       while (nr_retries && mem->res.usage > 0) {
+       while (nr_retries && memcg->res.usage > 0) {
                int progress;
 
                if (signal_pending(current)) {
                        ret = -EINTR;
                        goto out;
                }
-               progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
-                                               false, get_swappiness(mem));
+               progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
+                                               false);
                if (!progress) {
                        nr_retries--;
                        /* maybe some writeback is necessary */
@@ -3468,12 +3753,12 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
                                        u64 val)
 {
        int retval = 0;
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
        struct cgroup *parent = cont->parent;
-       struct mem_cgroup *parent_mem = NULL;
+       struct mem_cgroup *parent_memcg = NULL;
 
        if (parent)
-               parent_mem = mem_cgroup_from_cont(parent);
+               parent_memcg = mem_cgroup_from_cont(parent);
 
        cgroup_lock();
        /*
@@ -3484,10 +3769,10 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
         * For the root cgroup, parent_mem is NULL, we allow value to be
         * set if there are no children.
         */
-       if ((!parent_mem || !parent_mem->use_hierarchy) &&
+       if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
                                (val == 1 || val == 0)) {
                if (list_empty(&cont->children))
-                       mem->use_hierarchy = val;
+                       memcg->use_hierarchy = val;
                else
                        retval = -EBUSY;
        } else
@@ -3498,14 +3783,14 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
 }
 
 
-static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
-                               enum mem_cgroup_stat_index idx)
+static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
+                                              enum mem_cgroup_stat_index idx)
 {
        struct mem_cgroup *iter;
-       s64 val = 0;
+       long val = 0;
 
-       /* each per cpu's value can be minus.Then, use s64 */
-       for_each_mem_cgroup_tree(iter, mem)
+       /* Per-cpu values can be negative, use a signed accumulator */
+       for_each_mem_cgroup_tree(iter, memcg)
                val += mem_cgroup_read_stat(iter, idx);
 
        if (val < 0) /* race ? */
@@ -3513,30 +3798,29 @@ static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
        return val;
 }
 
-static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
+static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
 {
        u64 val;
 
-       if (!mem_cgroup_is_root(mem)) {
+       if (!mem_cgroup_is_root(memcg)) {
                if (!swap)
-                       return res_counter_read_u64(&mem->res, RES_USAGE);
+                       return res_counter_read_u64(&memcg->res, RES_USAGE);
                else
-                       return res_counter_read_u64(&mem->memsw, RES_USAGE);
+                       return res_counter_read_u64(&memcg->memsw, RES_USAGE);
        }
 
-       val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
-       val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
+       val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
+       val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
 
        if (swap)
-               val += mem_cgroup_get_recursive_idx_stat(mem,
-                               MEM_CGROUP_STAT_SWAPOUT);
+               val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
 
        return val << PAGE_SHIFT;
 }
 
 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
        u64 val;
        int type, name;
 
@@ -3545,15 +3829,15 @@ static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
        switch (type) {
        case _MEM:
                if (name == RES_USAGE)
-                       val = mem_cgroup_usage(mem, false);
+                       val = mem_cgroup_usage(memcg, false);
                else
-                       val = res_counter_read_u64(&mem->res, name);
+                       val = res_counter_read_u64(&memcg->res, name);
                break;
        case _MEMSWAP:
                if (name == RES_USAGE)
-                       val = mem_cgroup_usage(mem, true);
+                       val = mem_cgroup_usage(memcg, true);
                else
-                       val = res_counter_read_u64(&mem->memsw, name);
+                       val = res_counter_read_u64(&memcg->memsw, name);
                break;
        default:
                BUG();
@@ -3641,24 +3925,24 @@ out:
 
 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        int type, name;
 
-       mem = mem_cgroup_from_cont(cont);
+       memcg = mem_cgroup_from_cont(cont);
        type = MEMFILE_TYPE(event);
        name = MEMFILE_ATTR(event);
        switch (name) {
        case RES_MAX_USAGE:
                if (type == _MEM)
-                       res_counter_reset_max(&mem->res);
+                       res_counter_reset_max(&memcg->res);
                else
-                       res_counter_reset_max(&mem->memsw);
+                       res_counter_reset_max(&memcg->memsw);
                break;
        case RES_FAILCNT:
                if (type == _MEM)
-                       res_counter_reset_failcnt(&mem->res);
+                       res_counter_reset_failcnt(&memcg->res);
                else
-                       res_counter_reset_failcnt(&mem->memsw);
+                       res_counter_reset_failcnt(&memcg->memsw);
                break;
        }
 
@@ -3675,7 +3959,7 @@ static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
                                        struct cftype *cft, u64 val)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
        if (val >= (1 << NR_MOVE_TYPE))
                return -EINVAL;
@@ -3685,7 +3969,7 @@ static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
         * inconsistent.
         */
        cgroup_lock();
-       mem->move_charge_at_immigrate = val;
+       memcg->move_charge_at_immigrate = val;
        cgroup_unlock();
 
        return 0;
@@ -3707,6 +3991,8 @@ enum {
        MCS_PGPGIN,
        MCS_PGPGOUT,
        MCS_SWAP,
+       MCS_PGFAULT,
+       MCS_PGMAJFAULT,
        MCS_INACTIVE_ANON,
        MCS_ACTIVE_ANON,
        MCS_INACTIVE_FILE,
@@ -3729,6 +4015,8 @@ struct {
        {"pgpgin", "total_pgpgin"},
        {"pgpgout", "total_pgpgout"},
        {"swap", "total_swap"},
+       {"pgfault", "total_pgfault"},
+       {"pgmajfault", "total_pgmajfault"},
        {"inactive_anon", "total_inactive_anon"},
        {"active_anon", "total_active_anon"},
        {"inactive_file", "total_inactive_file"},
@@ -3738,48 +4026,99 @@ struct {
 
 
 static void
-mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
+mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 {
        s64 val;
 
        /* per cpu stat */
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_CACHE);
        s->stat[MCS_CACHE] += val * PAGE_SIZE;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_RSS);
        s->stat[MCS_RSS] += val * PAGE_SIZE;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
        s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGIN);
        s->stat[MCS_PGPGIN] += val;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGOUT);
        s->stat[MCS_PGPGOUT] += val;
        if (do_swap_account) {
-               val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
+               val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
                s->stat[MCS_SWAP] += val * PAGE_SIZE;
        }
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGFAULT);
+       s->stat[MCS_PGFAULT] += val;
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT);
+       s->stat[MCS_PGMAJFAULT] += val;
 
        /* per zone stat */
-       val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
        s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));
        s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
        s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));
        s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
        s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
 }
 
 static void
-mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
+mem_cgroup_get_total_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 {
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                mem_cgroup_get_local_stat(iter, s);
 }
 
+#ifdef CONFIG_NUMA
+static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
+{
+       int nid;
+       unsigned long total_nr, file_nr, anon_nr, unevictable_nr;
+       unsigned long node_nr;
+       struct cgroup *cont = m->private;
+       struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
+
+       total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
+       seq_printf(m, "total=%lu", total_nr);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
+               seq_printf(m, " N%d=%lu", nid, node_nr);
+       }
+       seq_putc(m, '\n');
+
+       file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
+       seq_printf(m, "file=%lu", file_nr);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               LRU_ALL_FILE);
+               seq_printf(m, " N%d=%lu", nid, node_nr);
+       }
+       seq_putc(m, '\n');
+
+       anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
+       seq_printf(m, "anon=%lu", anon_nr);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               LRU_ALL_ANON);
+               seq_printf(m, " N%d=%lu", nid, node_nr);
+       }
+       seq_putc(m, '\n');
+
+       unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
+       seq_printf(m, "unevictable=%lu", unevictable_nr);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               BIT(LRU_UNEVICTABLE));
+               seq_printf(m, " N%d=%lu", nid, node_nr);
+       }
+       seq_putc(m, '\n');
+       return 0;
+}
+#endif /* CONFIG_NUMA */
+
 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
                                 struct cgroup_map_cb *cb)
 {
@@ -3790,6 +4129,7 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
        memset(&mystat, 0, sizeof(mystat));
        mem_cgroup_get_local_stat(mem_cont, &mystat);
 
+
        for (i = 0; i < NR_MCS_STAT; i++) {
                if (i == MCS_SWAP && !do_swap_account)
                        continue;
@@ -3814,8 +4154,6 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
        }
 
 #ifdef CONFIG_DEBUG_VM
-       cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
-
        {
                int nid, zid;
                struct mem_cgroup_per_zone *mz;
@@ -3849,7 +4187,7 @@ static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
 {
        struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
-       return get_swappiness(memcg);
+       return mem_cgroup_swappiness(memcg);
 }
 
 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
@@ -3875,9 +4213,7 @@ static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
                return -EINVAL;
        }
 
-       spin_lock(&memcg->reclaim_param_lock);
        memcg->swappiness = val;
-       spin_unlock(&memcg->reclaim_param_lock);
 
        cgroup_unlock();
 
@@ -3954,20 +4290,20 @@ static int compare_thresholds(const void *a, const void *b)
        return _a->threshold - _b->threshold;
 }
 
-static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
+static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
 {
        struct mem_cgroup_eventfd_list *ev;
 
-       list_for_each_entry(ev, &mem->oom_notify, list)
+       list_for_each_entry(ev, &memcg->oom_notify, list)
                eventfd_signal(ev->eventfd, 1);
        return 0;
 }
 
-static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
+static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                mem_cgroup_oom_notify_cb(iter);
 }
 
@@ -4141,15 +4477,15 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
        if (!event)
                return -ENOMEM;
 
-       mutex_lock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
 
        event->eventfd = eventfd;
        list_add(&event->list, &memcg->oom_notify);
 
        /* already in OOM ? */
-       if (atomic_read(&memcg->oom_lock))
+       if (atomic_read(&memcg->under_oom))
                eventfd_signal(eventfd, 1);
-       mutex_unlock(&memcg_oom_mutex);
+       spin_unlock(&memcg_oom_lock);
 
        return 0;
 }
@@ -4157,32 +4493,32 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
        struct cftype *cft, struct eventfd_ctx *eventfd)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
        struct mem_cgroup_eventfd_list *ev, *tmp;
        int type = MEMFILE_TYPE(cft->private);
 
        BUG_ON(type != _OOM_TYPE);
 
-       mutex_lock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
 
-       list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
+       list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
                if (ev->eventfd == eventfd) {
                        list_del(&ev->list);
                        kfree(ev);
                }
        }
 
-       mutex_unlock(&memcg_oom_mutex);
+       spin_unlock(&memcg_oom_lock);
 }
 
 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
        struct cftype *cft,  struct cgroup_map_cb *cb)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
-       cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
+       cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
 
-       if (atomic_read(&mem->oom_lock))
+       if (atomic_read(&memcg->under_oom))
                cb->fill(cb, "under_oom", 1);
        else
                cb->fill(cb, "under_oom", 0);
@@ -4192,7 +4528,7 @@ static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
        struct cftype *cft, u64 val)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
        struct mem_cgroup *parent;
 
        /* cannot set to root cgroup and only 0 and 1 are allowed */
@@ -4204,17 +4540,63 @@ static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
        cgroup_lock();
        /* oom-kill-disable is a flag for subhierarchy. */
        if ((parent->use_hierarchy) ||
-           (mem->use_hierarchy && !list_empty(&cgrp->children))) {
+           (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
                cgroup_unlock();
                return -EINVAL;
        }
-       mem->oom_kill_disable = val;
+       memcg->oom_kill_disable = val;
        if (!val)
-               memcg_oom_recover(mem);
+               memcg_oom_recover(memcg);
        cgroup_unlock();
        return 0;
 }
 
+#ifdef CONFIG_NUMA
+static const struct file_operations mem_control_numa_stat_file_operations = {
+       .read = seq_read,
+       .llseek = seq_lseek,
+       .release = single_release,
+};
+
+static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
+{
+       struct cgroup *cont = file->f_dentry->d_parent->d_fsdata;
+
+       file->f_op = &mem_control_numa_stat_file_operations;
+       return single_open(file, mem_control_numa_stat_show, cont);
+}
+#endif /* CONFIG_NUMA */
+
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
+static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+{
+       /*
+        * Part of this would be better living in a separate allocation
+        * function, leaving us with just the cgroup tree population work.
+        * We, however, depend on state such as network's proto_list that
+        * is only initialized after cgroup creation. I found the less
+        * cumbersome way to deal with it to defer it all to populate time
+        */
+       return mem_cgroup_sockets_init(cont, ss);
+};
+
+static void kmem_cgroup_destroy(struct cgroup_subsys *ss,
+                               struct cgroup *cont)
+{
+       mem_cgroup_sockets_destroy(cont, ss);
+}
+#else
+static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+{
+       return 0;
+}
+
+static void kmem_cgroup_destroy(struct cgroup_subsys *ss,
+                               struct cgroup *cont)
+{
+}
+#endif
+
 static struct cftype mem_cgroup_files[] = {
        {
                .name = "usage_in_bytes",
@@ -4278,6 +4660,13 @@ static struct cftype mem_cgroup_files[] = {
                .unregister_event = mem_cgroup_oom_unregister_event,
                .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
        },
+#ifdef CONFIG_NUMA
+       {
+               .name = "numa_stat",
+               .open = mem_control_numa_stat_open,
+               .mode = S_IRUGO,
+       },
+#endif
 };
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
@@ -4323,7 +4712,7 @@ static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
 }
 #endif
 
-static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
        struct mem_cgroup_per_node *pn;
        struct mem_cgroup_per_zone *mz;
@@ -4343,21 +4732,21 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
        if (!pn)
                return 1;
 
-       mem->info.nodeinfo[node] = pn;
        for (zone = 0; zone < MAX_NR_ZONES; zone++) {
                mz = &pn->zoneinfo[zone];
                for_each_lru(l)
-                       INIT_LIST_HEAD(&mz->lists[l]);
+                       INIT_LIST_HEAD(&mz->lruvec.lists[l]);
                mz->usage_in_excess = 0;
                mz->on_tree = false;
-               mz->mem = mem;
+               mz->mem = memcg;
        }
+       memcg->info.nodeinfo[node] = pn;
        return 0;
 }
 
-static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
-       kfree(mem->info.nodeinfo[node]);
+       kfree(memcg->info.nodeinfo[node]);
 }
 
 static struct mem_cgroup *mem_cgroup_alloc(void)
@@ -4399,52 +4788,53 @@ out_free:
  * Removal of cgroup itself succeeds regardless of refs from swap.
  */
 
-static void __mem_cgroup_free(struct mem_cgroup *mem)
+static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
        int node;
 
-       mem_cgroup_remove_from_trees(mem);
-       free_css_id(&mem_cgroup_subsys, &mem->css);
+       mem_cgroup_remove_from_trees(memcg);
+       free_css_id(&mem_cgroup_subsys, &memcg->css);
 
-       for_each_node_state(node, N_POSSIBLE)
-               free_mem_cgroup_per_zone_info(mem, node);
+       for_each_node(node)
+               free_mem_cgroup_per_zone_info(memcg, node);
 
-       free_percpu(mem->stat);
+       free_percpu(memcg->stat);
        if (sizeof(struct mem_cgroup) < PAGE_SIZE)
-               kfree(mem);
+               kfree(memcg);
        else
-               vfree(mem);
+               vfree(memcg);
 }
 
-static void mem_cgroup_get(struct mem_cgroup *mem)
+static void mem_cgroup_get(struct mem_cgroup *memcg)
 {
-       atomic_inc(&mem->refcnt);
+       atomic_inc(&memcg->refcnt);
 }
 
-static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
+static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
 {
-       if (atomic_sub_and_test(count, &mem->refcnt)) {
-               struct mem_cgroup *parent = parent_mem_cgroup(mem);
-               __mem_cgroup_free(mem);
+       if (atomic_sub_and_test(count, &memcg->refcnt)) {
+               struct mem_cgroup *parent = parent_mem_cgroup(memcg);
+               __mem_cgroup_free(memcg);
                if (parent)
                        mem_cgroup_put(parent);
        }
 }
 
-static void mem_cgroup_put(struct mem_cgroup *mem)
+static void mem_cgroup_put(struct mem_cgroup *memcg)
 {
-       __mem_cgroup_put(mem, 1);
+       __mem_cgroup_put(memcg, 1);
 }
 
 /*
  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  */
-static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
+struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
 {
-       if (!mem->res.parent)
+       if (!memcg->res.parent)
                return NULL;
-       return mem_cgroup_from_res_counter(mem->res.parent, res);
+       return mem_cgroup_from_res_counter(memcg->res.parent, res);
 }
+EXPORT_SYMBOL(parent_mem_cgroup);
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 static void __init enable_swap_cgroup(void)
@@ -4464,13 +4854,13 @@ static int mem_cgroup_soft_limit_tree_init(void)
        struct mem_cgroup_tree_per_zone *rtpz;
        int tmp, node, zone;
 
-       for_each_node_state(node, N_POSSIBLE) {
+       for_each_node(node) {
                tmp = node;
                if (!node_state(node, N_NORMAL_MEMORY))
                        tmp = -1;
                rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
                if (!rtpn)
-                       return 1;
+                       goto err_cleanup;
 
                soft_limit_tree.rb_tree_per_node[node] = rtpn;
 
@@ -4481,21 +4871,31 @@ static int mem_cgroup_soft_limit_tree_init(void)
                }
        }
        return 0;
+
+err_cleanup:
+       for_each_node(node) {
+               if (!soft_limit_tree.rb_tree_per_node[node])
+                       break;
+               kfree(soft_limit_tree.rb_tree_per_node[node]);
+               soft_limit_tree.rb_tree_per_node[node] = NULL;
+       }
+       return 1;
+
 }
 
 static struct cgroup_subsys_state * __ref
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
-       struct mem_cgroup *mem, *parent;
+       struct mem_cgroup *memcg, *parent;
        long error = -ENOMEM;
        int node;
 
-       mem = mem_cgroup_alloc();
-       if (!mem)
+       memcg = mem_cgroup_alloc();
+       if (!memcg)
                return ERR_PTR(error);
 
-       for_each_node_state(node, N_POSSIBLE)
-               if (alloc_mem_cgroup_per_zone_info(mem, node))
+       for_each_node(node)
+               if (alloc_mem_cgroup_per_zone_info(memcg, node))
                        goto free_out;
 
        /* root ? */
@@ -4503,9 +4903,9 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                int cpu;
                enable_swap_cgroup();
                parent = NULL;
-               root_mem_cgroup = mem;
                if (mem_cgroup_soft_limit_tree_init())
                        goto free_out;
+               root_mem_cgroup = memcg;
                for_each_possible_cpu(cpu) {
                        struct memcg_stock_pcp *stock =
                                                &per_cpu(memcg_stock, cpu);
@@ -4514,13 +4914,13 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
        } else {
                parent = mem_cgroup_from_cont(cont->parent);
-               mem->use_hierarchy = parent->use_hierarchy;
-               mem->oom_kill_disable = parent->oom_kill_disable;
+               memcg->use_hierarchy = parent->use_hierarchy;
+               memcg->oom_kill_disable = parent->oom_kill_disable;
        }
 
        if (parent && parent->use_hierarchy) {
-               res_counter_init(&mem->res, &parent->res);
-               res_counter_init(&mem->memsw, &parent->memsw);
+               res_counter_init(&memcg->res, &parent->res);
+               res_counter_init(&memcg->memsw, &parent->memsw);
                /*
                 * We increment refcnt of the parent to ensure that we can
                 * safely access it on res_counter_charge/uncharge.
@@ -4529,39 +4929,39 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                 */
                mem_cgroup_get(parent);
        } else {
-               res_counter_init(&mem->res, NULL);
-               res_counter_init(&mem->memsw, NULL);
+               res_counter_init(&memcg->res, NULL);
+               res_counter_init(&memcg->memsw, NULL);
        }
-       mem->last_scanned_child = 0;
-       spin_lock_init(&mem->reclaim_param_lock);
-       INIT_LIST_HEAD(&mem->oom_notify);
+       memcg->last_scanned_node = MAX_NUMNODES;
+       INIT_LIST_HEAD(&memcg->oom_notify);
 
        if (parent)
-               mem->swappiness = get_swappiness(parent);
-       atomic_set(&mem->refcnt, 1);
-       mem->move_charge_at_immigrate = 0;
-       mutex_init(&mem->thresholds_lock);
-       return &mem->css;
+               memcg->swappiness = mem_cgroup_swappiness(parent);
+       atomic_set(&memcg->refcnt, 1);
+       memcg->move_charge_at_immigrate = 0;
+       mutex_init(&memcg->thresholds_lock);
+       return &memcg->css;
 free_out:
-       __mem_cgroup_free(mem);
-       root_mem_cgroup = NULL;
+       __mem_cgroup_free(memcg);
        return ERR_PTR(error);
 }
 
 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
                                        struct cgroup *cont)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
 
-       return mem_cgroup_force_empty(mem, false);
+       return mem_cgroup_force_empty(memcg, false);
 }
 
 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
                                struct cgroup *cont)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
+
+       kmem_cgroup_destroy(ss, cont);
 
-       mem_cgroup_put(mem);
+       mem_cgroup_put(memcg);
 }
 
 static int mem_cgroup_populate(struct cgroup_subsys *ss,
@@ -4574,6 +4974,10 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss,
 
        if (!ret)
                ret = register_memsw_files(cont, ss);
+
+       if (!ret)
+               ret = register_kmem_files(cont, ss);
+
        return ret;
 }
 
@@ -4584,9 +4988,9 @@ static int mem_cgroup_do_precharge(unsigned long count)
 {
        int ret = 0;
        int batch_count = PRECHARGE_COUNT_AT_ONCE;
-       struct mem_cgroup *mem = mc.to;
+       struct mem_cgroup *memcg = mc.to;
 
-       if (mem_cgroup_is_root(mem)) {
+       if (mem_cgroup_is_root(memcg)) {
                mc.precharge += count;
                /* we don't need css_get for root */
                return ret;
@@ -4595,16 +4999,16 @@ static int mem_cgroup_do_precharge(unsigned long count)
        if (count > 1) {
                struct res_counter *dummy;
                /*
-                * "mem" cannot be under rmdir() because we've already checked
+                * "memcg" cannot be under rmdir() because we've already checked
                 * by cgroup_lock_live_cgroup() that it is not removed and we
                 * are still under the same cgroup_mutex. So we can postpone
                 * css_get().
                 */
-               if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
+               if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
                        goto one_by_one;
-               if (do_swap_account && res_counter_charge(&mem->memsw,
+               if (do_swap_account && res_counter_charge(&memcg->memsw,
                                                PAGE_SIZE * count, &dummy)) {
-                       res_counter_uncharge(&mem->res, PAGE_SIZE * count);
+                       res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
                        goto one_by_one;
                }
                mc.precharge += count;
@@ -4621,11 +5025,11 @@ one_by_one:
                        batch_count = PRECHARGE_COUNT_AT_ONCE;
                        cond_resched();
                }
-               ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false,
-                                             PAGE_SIZE);
-               if (ret || !mem)
+               ret = __mem_cgroup_try_charge(NULL,
+                                       GFP_KERNEL, 1, &memcg, false);
+               if (ret)
                        /* mem_cgroup_clear_mc() will do uncharge later */
-                       return -ENOMEM;
+                       return ret;
                mc.precharge++;
        }
        return ret;
@@ -4722,15 +5126,17 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
                pgoff = pte_to_pgoff(ptent);
 
        /* page is moved even if it's not RSS of this task(page-faulted). */
-       if (!mapping_cap_swap_backed(mapping)) { /* normal file */
-               page = find_get_page(mapping, pgoff);
-       } else { /* shmem/tmpfs file. we should take account of swap too. */
-               swp_entry_t ent;
-               mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
+       page = find_get_page(mapping, pgoff);
+
+#ifdef CONFIG_SWAP
+       /* shmem/tmpfs may report page out on swap: account for that too. */
+       if (radix_tree_exceptional_entry(page)) {
+               swp_entry_t swap = radix_to_swp_entry(page);
                if (do_swap_account)
-                       entry->val = ent.val;
+                       *entry = swap;
+               page = find_get_page(&swapper_space, swap.val);
        }
-
+#endif
        return page;
 }
 
@@ -4768,7 +5174,7 @@ static int is_target_pte_for_mc(struct vm_area_struct *vma,
        }
        /* There is a swap entry and a page doesn't exist or isn't charged */
        if (ent.val && !ret &&
-                       css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
+                       css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
                ret = MC_TARGET_SWAP;
                if (target)
                        target->ent = ent;
@@ -4892,17 +5298,17 @@ static void mem_cgroup_clear_mc(void)
 
 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
+       struct task_struct *p = cgroup_taskset_first(tset);
        int ret = 0;
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
 
-       if (mem->move_charge_at_immigrate) {
+       if (memcg->move_charge_at_immigrate) {
                struct mm_struct *mm;
                struct mem_cgroup *from = mem_cgroup_from_task(p);
 
-               VM_BUG_ON(from == mem);
+               VM_BUG_ON(from == memcg);
 
                mm = get_task_mm(p);
                if (!mm)
@@ -4917,7 +5323,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                        mem_cgroup_start_move(from);
                        spin_lock(&mc.lock);
                        mc.from = from;
-                       mc.to = mem;
+                       mc.to = memcg;
                        spin_unlock(&mc.lock);
                        /* We set mc.moving_task later */
 
@@ -4932,8 +5338,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
 
 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
        mem_cgroup_clear_mc();
 }
@@ -4968,8 +5373,8 @@ retry:
                        if (isolate_lru_page(page))
                                goto put;
                        pc = lookup_page_cgroup(page);
-                       if (!mem_cgroup_move_account(pc,
-                                       mc.from, mc.to, false, PAGE_SIZE)) {
+                       if (!mem_cgroup_move_account(page, 1, pc,
+                                                    mc.from, mc.to, false)) {
                                mc.precharge--;
                                /* we uncharge from mc.from later. */
                                mc.moved_charge++;
@@ -5050,42 +5455,35 @@ retry:
 
 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                struct cgroup *cont,
-                               struct cgroup *old_cont,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
-       struct mm_struct *mm;
+       struct task_struct *p = cgroup_taskset_first(tset);
+       struct mm_struct *mm = get_task_mm(p);
 
-       if (!mc.to)
-               /* no need to move charge */
-               return;
-
-       mm = get_task_mm(p);
        if (mm) {
-               mem_cgroup_move_charge(mm);
+               if (mc.to)
+                       mem_cgroup_move_charge(mm);
+               put_swap_token(mm);
                mmput(mm);
        }
-       mem_cgroup_clear_mc();
+       if (mc.to)
+               mem_cgroup_clear_mc();
 }
 #else  /* !CONFIG_MMU */
 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
        return 0;
 }
 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
 }
 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                struct cgroup *cont,
-                               struct cgroup *old_cont,
-                               struct task_struct *p,
-                               bool threadgroup)
+                               struct cgroup_taskset *tset)
 {
 }
 #endif
@@ -5108,19 +5506,12 @@ struct cgroup_subsys mem_cgroup_subsys = {
 static int __init enable_swap_account(char *s)
 {
        /* consider enabled if no parameter or 1 is given */
-       if (!(*s) || !strcmp(s, "=1"))
+       if (!strcmp(s, "1"))
                really_do_swap_account = 1;
-       else if (!strcmp(s, "=0"))
+       else if (!strcmp(s, "0"))
                really_do_swap_account = 0;
        return 1;
 }
-__setup("swapaccount", enable_swap_account);
+__setup("swapaccount=", enable_swap_account);
 
-static int __init disable_swap_account(char *s)
-{
-       printk_once("noswapaccount is deprecated and will be removed in 2.6.40. Use swapaccount=0 instead\n");
-       enable_swap_account("=0");
-       return 1;
-}
-__setup("noswapaccount", disable_swap_account);
 #endif