drm/ttm: fix two regressions since move_notify changes
[linux-2.6.git] / mm / memcontrol.c
index 02a7947..3dbff4d 100644 (file)
@@ -33,6 +33,7 @@
 #include <linux/bit_spinlock.h>
 #include <linux/rcupdate.h>
 #include <linux/limits.h>
+#include <linux/export.h>
 #include <linux/mutex.h>
 #include <linux/rbtree.h>
 #include <linux/slab.h>
@@ -49,6 +50,8 @@
 #include <linux/cpu.h>
 #include <linux/oom.h>
 #include "internal.h"
+#include <net/sock.h>
+#include <net/tcp_memcontrol.h>
 
 #include <asm/uaccess.h>
 
@@ -107,10 +110,12 @@ enum mem_cgroup_events_index {
 enum mem_cgroup_events_target {
        MEM_CGROUP_TARGET_THRESH,
        MEM_CGROUP_TARGET_SOFTLIMIT,
+       MEM_CGROUP_TARGET_NUMAINFO,
        MEM_CGROUP_NTARGETS,
 };
 #define THRESHOLDS_EVENTS_TARGET (128)
 #define SOFTLIMIT_EVENTS_TARGET (1024)
+#define NUMAINFO_EVENTS_TARGET (1024)
 
 struct mem_cgroup_stat_cpu {
        long count[MEM_CGROUP_STAT_NSTATS];
@@ -118,16 +123,22 @@ struct mem_cgroup_stat_cpu {
        unsigned long targets[MEM_CGROUP_NTARGETS];
 };
 
+struct mem_cgroup_reclaim_iter {
+       /* css_id of the last scanned hierarchy member */
+       int position;
+       /* scan generation, increased every round-trip */
+       unsigned int generation;
+};
+
 /*
  * per-zone information in memory controller.
  */
 struct mem_cgroup_per_zone {
-       /*
-        * spin_lock to protect the per cgroup LRU
-        */
-       struct list_head        lists[NR_LRU_LISTS];
+       struct lruvec           lruvec;
        unsigned long           count[NR_LRU_LISTS];
 
+       struct mem_cgroup_reclaim_iter reclaim_iter[DEF_PRIORITY + 1];
+
        struct zone_reclaim_stat reclaim_stat;
        struct rb_node          tree_node;      /* RB tree node */
        unsigned long long      usage_in_excess;/* Set to the value by which */
@@ -199,8 +210,8 @@ struct mem_cgroup_eventfd_list {
        struct eventfd_ctx *eventfd;
 };
 
-static void mem_cgroup_threshold(struct mem_cgroup *mem);
-static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
+static void mem_cgroup_threshold(struct mem_cgroup *memcg);
+static void mem_cgroup_oom_notify(struct mem_cgroup *memcg);
 
 /*
  * The memory controller data structure. The memory controller controls both
@@ -228,24 +239,23 @@ struct mem_cgroup {
         * per zone LRU lists.
         */
        struct mem_cgroup_lru_info info;
-       /*
-        * While reclaiming in a hierarchy, we cache the last child we
-        * reclaimed from.
-        */
-       int last_scanned_child;
        int last_scanned_node;
 #if MAX_NUMNODES > 1
        nodemask_t      scan_nodes;
-       unsigned long   next_scan_node_update;
+       atomic_t        numainfo_events;
+       atomic_t        numainfo_updating;
 #endif
        /*
         * Should the accounting and control be hierarchical, per subtree?
         */
        bool use_hierarchy;
-       atomic_t        oom_lock;
+
+       bool            oom_lock;
+       atomic_t        under_oom;
+
        atomic_t        refcnt;
 
-       unsigned int    swappiness;
+       int     swappiness;
        /* OOM-Killer disable */
        int             oom_kill_disable;
 
@@ -279,6 +289,10 @@ struct mem_cgroup {
         */
        struct mem_cgroup_stat_cpu nocpu_base;
        spinlock_t pcp_counter_lock;
+
+#ifdef CONFIG_INET
+       struct tcp_memcontrol tcp_mem;
+#endif
 };
 
 /* Stuffs for move charges at task migration. */
@@ -353,32 +367,90 @@ enum charge_type {
 #define MEM_CGROUP_RECLAIM_NOSWAP      (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
 #define MEM_CGROUP_RECLAIM_SHRINK_BIT  0x1
 #define MEM_CGROUP_RECLAIM_SHRINK      (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
-#define MEM_CGROUP_RECLAIM_SOFT_BIT    0x2
-#define MEM_CGROUP_RECLAIM_SOFT                (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
 
-static void mem_cgroup_get(struct mem_cgroup *mem);
-static void mem_cgroup_put(struct mem_cgroup *mem);
-static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
-static void drain_all_stock_async(void);
+static void mem_cgroup_get(struct mem_cgroup *memcg);
+static void mem_cgroup_put(struct mem_cgroup *memcg);
+
+/* Writing them here to avoid exposing memcg's inner layout */
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
+#include <net/sock.h>
+#include <net/ip.h>
+
+static bool mem_cgroup_is_root(struct mem_cgroup *memcg);
+void sock_update_memcg(struct sock *sk)
+{
+       if (static_branch(&memcg_socket_limit_enabled)) {
+               struct mem_cgroup *memcg;
+
+               BUG_ON(!sk->sk_prot->proto_cgroup);
+
+               /* Socket cloning can throw us here with sk_cgrp already
+                * filled. It won't however, necessarily happen from
+                * process context. So the test for root memcg given
+                * the current task's memcg won't help us in this case.
+                *
+                * Respecting the original socket's memcg is a better
+                * decision in this case.
+                */
+               if (sk->sk_cgrp) {
+                       BUG_ON(mem_cgroup_is_root(sk->sk_cgrp->memcg));
+                       mem_cgroup_get(sk->sk_cgrp->memcg);
+                       return;
+               }
+
+               rcu_read_lock();
+               memcg = mem_cgroup_from_task(current);
+               if (!mem_cgroup_is_root(memcg)) {
+                       mem_cgroup_get(memcg);
+                       sk->sk_cgrp = sk->sk_prot->proto_cgroup(memcg);
+               }
+               rcu_read_unlock();
+       }
+}
+EXPORT_SYMBOL(sock_update_memcg);
+
+void sock_release_memcg(struct sock *sk)
+{
+       if (static_branch(&memcg_socket_limit_enabled) && sk->sk_cgrp) {
+               struct mem_cgroup *memcg;
+               WARN_ON(!sk->sk_cgrp->memcg);
+               memcg = sk->sk_cgrp->memcg;
+               mem_cgroup_put(memcg);
+       }
+}
+
+#ifdef CONFIG_INET
+struct cg_proto *tcp_proto_cgroup(struct mem_cgroup *memcg)
+{
+       if (!memcg || mem_cgroup_is_root(memcg))
+               return NULL;
+
+       return &memcg->tcp_mem.cg_proto;
+}
+EXPORT_SYMBOL(tcp_proto_cgroup);
+#endif /* CONFIG_INET */
+#endif /* CONFIG_CGROUP_MEM_RES_CTLR_KMEM */
+
+static void drain_all_stock_async(struct mem_cgroup *memcg);
 
 static struct mem_cgroup_per_zone *
-mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
+mem_cgroup_zoneinfo(struct mem_cgroup *memcg, int nid, int zid)
 {
-       return &mem->info.nodeinfo[nid]->zoneinfo[zid];
+       return &memcg->info.nodeinfo[nid]->zoneinfo[zid];
 }
 
-struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
+struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *memcg)
 {
-       return &mem->css;
+       return &memcg->css;
 }
 
 static struct mem_cgroup_per_zone *
-page_cgroup_zoneinfo(struct mem_cgroup *mem, struct page *page)
+page_cgroup_zoneinfo(struct mem_cgroup *memcg, struct page *page)
 {
        int nid = page_to_nid(page);
        int zid = page_zonenum(page);
 
-       return mem_cgroup_zoneinfo(mem, nid, zid);
+       return mem_cgroup_zoneinfo(memcg, nid, zid);
 }
 
 static struct mem_cgroup_tree_per_zone *
@@ -397,7 +469,7 @@ soft_limit_tree_from_page(struct page *page)
 }
 
 static void
-__mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
+__mem_cgroup_insert_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz,
                                unsigned long long new_usage_in_excess)
@@ -431,7 +503,7 @@ __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
 }
 
 static void
-__mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+__mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz)
 {
@@ -442,17 +514,17 @@ __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
 }
 
 static void
-mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
+mem_cgroup_remove_exceeded(struct mem_cgroup *memcg,
                                struct mem_cgroup_per_zone *mz,
                                struct mem_cgroup_tree_per_zone *mctz)
 {
        spin_lock(&mctz->lock);
-       __mem_cgroup_remove_exceeded(mem, mz, mctz);
+       __mem_cgroup_remove_exceeded(memcg, mz, mctz);
        spin_unlock(&mctz->lock);
 }
 
 
-static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
+static void mem_cgroup_update_tree(struct mem_cgroup *memcg, struct page *page)
 {
        unsigned long long excess;
        struct mem_cgroup_per_zone *mz;
@@ -465,9 +537,9 @@ static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
         * Necessary to update all ancestors when hierarchy is used.
         * because their event counter is not touched.
         */
-       for (; mem; mem = parent_mem_cgroup(mem)) {
-               mz = mem_cgroup_zoneinfo(mem, nid, zid);
-               excess = res_counter_soft_limit_excess(&mem->res);
+       for (; memcg; memcg = parent_mem_cgroup(memcg)) {
+               mz = mem_cgroup_zoneinfo(memcg, nid, zid);
+               excess = res_counter_soft_limit_excess(&memcg->res);
                /*
                 * We have to update the tree if mz is on RB-tree or
                 * mem is over its softlimit.
@@ -476,28 +548,28 @@ static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
                        spin_lock(&mctz->lock);
                        /* if on-tree, remove it */
                        if (mz->on_tree)
-                               __mem_cgroup_remove_exceeded(mem, mz, mctz);
+                               __mem_cgroup_remove_exceeded(memcg, mz, mctz);
                        /*
                         * Insert again. mz->usage_in_excess will be updated.
                         * If excess is 0, no tree ops.
                         */
-                       __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
+                       __mem_cgroup_insert_exceeded(memcg, mz, mctz, excess);
                        spin_unlock(&mctz->lock);
                }
        }
 }
 
-static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
+static void mem_cgroup_remove_from_trees(struct mem_cgroup *memcg)
 {
        int node, zone;
        struct mem_cgroup_per_zone *mz;
        struct mem_cgroup_tree_per_zone *mctz;
 
-       for_each_node_state(node, N_POSSIBLE) {
+       for_each_node(node) {
                for (zone = 0; zone < MAX_NR_ZONES; zone++) {
-                       mz = mem_cgroup_zoneinfo(mem, node, zone);
+                       mz = mem_cgroup_zoneinfo(memcg, node, zone);
                        mctz = soft_limit_tree_node_zone(node, zone);
-                       mem_cgroup_remove_exceeded(mem, mz, mctz);
+                       mem_cgroup_remove_exceeded(memcg, mz, mctz);
                }
        }
 }
@@ -558,7 +630,7 @@ mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
  * common workload, threashold and synchonization as vmstat[] should be
  * implemented.
  */
-static long mem_cgroup_read_stat(struct mem_cgroup *mem,
+static long mem_cgroup_read_stat(struct mem_cgroup *memcg,
                                 enum mem_cgroup_stat_index idx)
 {
        long val = 0;
@@ -566,155 +638,166 @@ static long mem_cgroup_read_stat(struct mem_cgroup *mem,
 
        get_online_cpus();
        for_each_online_cpu(cpu)
-               val += per_cpu(mem->stat->count[idx], cpu);
+               val += per_cpu(memcg->stat->count[idx], cpu);
 #ifdef CONFIG_HOTPLUG_CPU
-       spin_lock(&mem->pcp_counter_lock);
-       val += mem->nocpu_base.count[idx];
-       spin_unlock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
+       val += memcg->nocpu_base.count[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
 #endif
        put_online_cpus();
        return val;
 }
 
-static long mem_cgroup_local_usage(struct mem_cgroup *mem)
-{
-       long ret;
-
-       ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
-       ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
-       return ret;
-}
-
-static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
+static void mem_cgroup_swap_statistics(struct mem_cgroup *memcg,
                                         bool charge)
 {
        int val = (charge) ? 1 : -1;
-       this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
+       this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
 }
 
-void mem_cgroup_pgfault(struct mem_cgroup *mem, int val)
-{
-       this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGFAULT], val);
-}
-
-void mem_cgroup_pgmajfault(struct mem_cgroup *mem, int val)
-{
-       this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT], val);
-}
-
-static unsigned long mem_cgroup_read_events(struct mem_cgroup *mem,
+static unsigned long mem_cgroup_read_events(struct mem_cgroup *memcg,
                                            enum mem_cgroup_events_index idx)
 {
        unsigned long val = 0;
        int cpu;
 
        for_each_online_cpu(cpu)
-               val += per_cpu(mem->stat->events[idx], cpu);
+               val += per_cpu(memcg->stat->events[idx], cpu);
 #ifdef CONFIG_HOTPLUG_CPU
-       spin_lock(&mem->pcp_counter_lock);
-       val += mem->nocpu_base.events[idx];
-       spin_unlock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
+       val += memcg->nocpu_base.events[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
 #endif
        return val;
 }
 
-static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
+static void mem_cgroup_charge_statistics(struct mem_cgroup *memcg,
                                         bool file, int nr_pages)
 {
        preempt_disable();
 
        if (file)
-               __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], nr_pages);
+               __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_CACHE],
+                               nr_pages);
        else
-               __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], nr_pages);
+               __this_cpu_add(memcg->stat->count[MEM_CGROUP_STAT_RSS],
+                               nr_pages);
 
        /* pagein of a big page is an event. So, ignore page size */
        if (nr_pages > 0)
-               __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
+               __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGIN]);
        else {
-               __this_cpu_inc(mem->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
+               __this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGPGOUT]);
                nr_pages = -nr_pages; /* for event */
        }
 
-       __this_cpu_add(mem->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
+       __this_cpu_add(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT], nr_pages);
 
        preempt_enable();
 }
 
-static unsigned long
-mem_cgroup_get_zonestat_node(struct mem_cgroup *mem, int nid, enum lru_list idx)
+unsigned long
+mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg, int nid, int zid,
+                       unsigned int lru_mask)
 {
        struct mem_cgroup_per_zone *mz;
-       u64 total = 0;
-       int zid;
+       enum lru_list l;
+       unsigned long ret = 0;
+
+       mz = mem_cgroup_zoneinfo(memcg, nid, zid);
 
-       for (zid = 0; zid < MAX_NR_ZONES; zid++) {
-               mz = mem_cgroup_zoneinfo(mem, nid, zid);
-               total += MEM_CGROUP_ZSTAT(mz, idx);
+       for_each_lru(l) {
+               if (BIT(l) & lru_mask)
+                       ret += MEM_CGROUP_ZSTAT(mz, l);
        }
-       return total;
+       return ret;
 }
-static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
-                                       enum lru_list idx)
+
+static unsigned long
+mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
+                       int nid, unsigned int lru_mask)
 {
-       int nid;
        u64 total = 0;
+       int zid;
+
+       for (zid = 0; zid < MAX_NR_ZONES; zid++)
+               total += mem_cgroup_zone_nr_lru_pages(memcg,
+                                               nid, zid, lru_mask);
 
-       for_each_online_node(nid)
-               total += mem_cgroup_get_zonestat_node(mem, nid, idx);
        return total;
 }
 
-static bool __memcg_event_check(struct mem_cgroup *mem, int target)
+static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg,
+                       unsigned int lru_mask)
 {
-       unsigned long val, next;
+       int nid;
+       u64 total = 0;
 
-       val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
-       next = this_cpu_read(mem->stat->targets[target]);
-       /* from time_after() in jiffies.h */
-       return ((long)next - (long)val < 0);
+       for_each_node_state(nid, N_HIGH_MEMORY)
+               total += mem_cgroup_node_nr_lru_pages(memcg, nid, lru_mask);
+       return total;
 }
 
-static void __mem_cgroup_target_update(struct mem_cgroup *mem, int target)
+static bool mem_cgroup_event_ratelimit(struct mem_cgroup *memcg,
+                                      enum mem_cgroup_events_target target)
 {
        unsigned long val, next;
 
-       val = this_cpu_read(mem->stat->events[MEM_CGROUP_EVENTS_COUNT]);
-
-       switch (target) {
-       case MEM_CGROUP_TARGET_THRESH:
-               next = val + THRESHOLDS_EVENTS_TARGET;
-               break;
-       case MEM_CGROUP_TARGET_SOFTLIMIT:
-               next = val + SOFTLIMIT_EVENTS_TARGET;
-               break;
-       default:
-               return;
+       val = __this_cpu_read(memcg->stat->events[MEM_CGROUP_EVENTS_COUNT]);
+       next = __this_cpu_read(memcg->stat->targets[target]);
+       /* from time_after() in jiffies.h */
+       if ((long)next - (long)val < 0) {
+               switch (target) {
+               case MEM_CGROUP_TARGET_THRESH:
+                       next = val + THRESHOLDS_EVENTS_TARGET;
+                       break;
+               case MEM_CGROUP_TARGET_SOFTLIMIT:
+                       next = val + SOFTLIMIT_EVENTS_TARGET;
+                       break;
+               case MEM_CGROUP_TARGET_NUMAINFO:
+                       next = val + NUMAINFO_EVENTS_TARGET;
+                       break;
+               default:
+                       break;
+               }
+               __this_cpu_write(memcg->stat->targets[target], next);
+               return true;
        }
-
-       this_cpu_write(mem->stat->targets[target], next);
+       return false;
 }
 
 /*
  * Check events in order.
  *
  */
-static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
+static void memcg_check_events(struct mem_cgroup *memcg, struct page *page)
 {
+       preempt_disable();
        /* threshold event is triggered in finer grain than soft limit */
-       if (unlikely(__memcg_event_check(mem, MEM_CGROUP_TARGET_THRESH))) {
-               mem_cgroup_threshold(mem);
-               __mem_cgroup_target_update(mem, MEM_CGROUP_TARGET_THRESH);
-               if (unlikely(__memcg_event_check(mem,
-                       MEM_CGROUP_TARGET_SOFTLIMIT))){
-                       mem_cgroup_update_tree(mem, page);
-                       __mem_cgroup_target_update(mem,
-                               MEM_CGROUP_TARGET_SOFTLIMIT);
-               }
-       }
+       if (unlikely(mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_THRESH))) {
+               bool do_softlimit, do_numainfo;
+
+               do_softlimit = mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_SOFTLIMIT);
+#if MAX_NUMNODES > 1
+               do_numainfo = mem_cgroup_event_ratelimit(memcg,
+                                               MEM_CGROUP_TARGET_NUMAINFO);
+#endif
+               preempt_enable();
+
+               mem_cgroup_threshold(memcg);
+               if (unlikely(do_softlimit))
+                       mem_cgroup_update_tree(memcg, page);
+#if MAX_NUMNODES > 1
+               if (unlikely(do_numainfo))
+                       atomic_inc(&memcg->numainfo_events);
+#endif
+       } else
+               preempt_enable();
 }
 
-static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
+struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
 {
        return container_of(cgroup_subsys_state(cont,
                                mem_cgroup_subsys_id), struct mem_cgroup,
@@ -737,7 +820,7 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
 
 struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
 
        if (!mm)
                return NULL;
@@ -748,115 +831,148 @@ struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
         */
        rcu_read_lock();
        do {
-               mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
-               if (unlikely(!mem))
+               memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
+               if (unlikely(!memcg))
                        break;
-       } while (!css_tryget(&mem->css));
+       } while (!css_tryget(&memcg->css));
        rcu_read_unlock();
-       return mem;
+       return memcg;
 }
 
-/* The caller has to guarantee "mem" exists before calling this */
-static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
+/**
+ * mem_cgroup_iter - iterate over memory cgroup hierarchy
+ * @root: hierarchy root
+ * @prev: previously returned memcg, NULL on first invocation
+ * @reclaim: cookie for shared reclaim walks, NULL for full walks
+ *
+ * Returns references to children of the hierarchy below @root, or
+ * @root itself, or %NULL after a full round-trip.
+ *
+ * Caller must pass the return value in @prev on subsequent
+ * invocations for reference counting, or use mem_cgroup_iter_break()
+ * to cancel a hierarchy walk before the round-trip is complete.
+ *
+ * Reclaimers can specify a zone and a priority level in @reclaim to
+ * divide up the memcgs in the hierarchy among all concurrent
+ * reclaimers operating on the same zone and priority.
+ */
+struct mem_cgroup *mem_cgroup_iter(struct mem_cgroup *root,
+                                  struct mem_cgroup *prev,
+                                  struct mem_cgroup_reclaim_cookie *reclaim)
 {
-       struct cgroup_subsys_state *css;
-       int found;
+       struct mem_cgroup *memcg = NULL;
+       int id = 0;
 
-       if (!mem) /* ROOT cgroup has the smallest ID */
-               return root_mem_cgroup; /*css_put/get against root is ignored*/
-       if (!mem->use_hierarchy) {
-               if (css_tryget(&mem->css))
-                       return mem;
+       if (mem_cgroup_disabled())
                return NULL;
-       }
-       rcu_read_lock();
-       /*
-        * searching a memory cgroup which has the smallest ID under given
-        * ROOT cgroup. (ID >= 1)
-        */
-       css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
-       if (css && css_tryget(css))
-               mem = container_of(css, struct mem_cgroup, css);
-       else
-               mem = NULL;
-       rcu_read_unlock();
-       return mem;
-}
 
-static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
-                                       struct mem_cgroup *root,
-                                       bool cond)
-{
-       int nextid = css_id(&iter->css) + 1;
-       int found;
-       int hierarchy_used;
-       struct cgroup_subsys_state *css;
+       if (!root)
+               root = root_mem_cgroup;
 
-       hierarchy_used = iter->use_hierarchy;
+       if (prev && !reclaim)
+               id = css_id(&prev->css);
 
-       css_put(&iter->css);
-       /* If no ROOT, walk all, ignore hierarchy */
-       if (!cond || (root && !hierarchy_used))
-               return NULL;
+       if (prev && prev != root)
+               css_put(&prev->css);
 
-       if (!root)
-               root = root_mem_cgroup;
+       if (!root->use_hierarchy && root != root_mem_cgroup) {
+               if (prev)
+                       return NULL;
+               return root;
+       }
 
-       do {
-               iter = NULL;
-               rcu_read_lock();
+       while (!memcg) {
+               struct mem_cgroup_reclaim_iter *uninitialized_var(iter);
+               struct cgroup_subsys_state *css;
+
+               if (reclaim) {
+                       int nid = zone_to_nid(reclaim->zone);
+                       int zid = zone_idx(reclaim->zone);
+                       struct mem_cgroup_per_zone *mz;
+
+                       mz = mem_cgroup_zoneinfo(root, nid, zid);
+                       iter = &mz->reclaim_iter[reclaim->priority];
+                       if (prev && reclaim->generation != iter->generation)
+                               return NULL;
+                       id = iter->position;
+               }
 
-               css = css_get_next(&mem_cgroup_subsys, nextid,
-                               &root->css, &found);
-               if (css && css_tryget(css))
-                       iter = container_of(css, struct mem_cgroup, css);
+               rcu_read_lock();
+               css = css_get_next(&mem_cgroup_subsys, id + 1, &root->css, &id);
+               if (css) {
+                       if (css == &root->css || css_tryget(css))
+                               memcg = container_of(css,
+                                                    struct mem_cgroup, css);
+               } else
+                       id = 0;
                rcu_read_unlock();
-               /* If css is NULL, no more cgroups will be found */
-               nextid = found + 1;
-       } while (css && !iter);
 
-       return iter;
+               if (reclaim) {
+                       iter->position = id;
+                       if (!css)
+                               iter->generation++;
+                       else if (!prev && memcg)
+                               reclaim->generation = iter->generation;
+               }
+
+               if (prev && !css)
+                       return NULL;
+       }
+       return memcg;
+}
+
+/**
+ * mem_cgroup_iter_break - abort a hierarchy walk prematurely
+ * @root: hierarchy root
+ * @prev: last visited hierarchy member as returned by mem_cgroup_iter()
+ */
+void mem_cgroup_iter_break(struct mem_cgroup *root,
+                          struct mem_cgroup *prev)
+{
+       if (!root)
+               root = root_mem_cgroup;
+       if (prev && prev != root)
+               css_put(&prev->css);
 }
+
 /*
- * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
- * be careful that "break" loop is not allowed. We have reference count.
- * Instead of that modify "cond" to be false and "continue" to exit the loop.
+ * Iteration constructs for visiting all cgroups (under a tree).  If
+ * loops are exited prematurely (break), mem_cgroup_iter_break() must
+ * be used for reference counting.
  */
-#define for_each_mem_cgroup_tree_cond(iter, root, cond)        \
-       for (iter = mem_cgroup_start_loop(root);\
-            iter != NULL;\
-            iter = mem_cgroup_get_next(iter, root, cond))
-
-#define for_each_mem_cgroup_tree(iter, root) \
-       for_each_mem_cgroup_tree_cond(iter, root, true)
-
-#define for_each_mem_cgroup_all(iter) \
-       for_each_mem_cgroup_tree_cond(iter, NULL, true)
+#define for_each_mem_cgroup_tree(iter, root)           \
+       for (iter = mem_cgroup_iter(root, NULL, NULL);  \
+            iter != NULL;                              \
+            iter = mem_cgroup_iter(root, iter, NULL))
 
+#define for_each_mem_cgroup(iter)                      \
+       for (iter = mem_cgroup_iter(NULL, NULL, NULL);  \
+            iter != NULL;                              \
+            iter = mem_cgroup_iter(NULL, iter, NULL))
 
-static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
+static inline bool mem_cgroup_is_root(struct mem_cgroup *memcg)
 {
-       return (mem == root_mem_cgroup);
+       return (memcg == root_mem_cgroup);
 }
 
 void mem_cgroup_count_vm_event(struct mm_struct *mm, enum vm_event_item idx)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
 
        if (!mm)
                return;
 
        rcu_read_lock();
-       mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
-       if (unlikely(!mem))
+       memcg = mem_cgroup_from_task(rcu_dereference(mm->owner));
+       if (unlikely(!memcg))
                goto out;
 
        switch (idx) {
-       case PGMAJFAULT:
-               mem_cgroup_pgmajfault(mem, 1);
-               break;
        case PGFAULT:
-               mem_cgroup_pgfault(mem, 1);
+               this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGFAULT]);
+               break;
+       case PGMAJFAULT:
+               this_cpu_inc(memcg->stat->events[MEM_CGROUP_EVENTS_PGMAJFAULT]);
                break;
        default:
                BUG();
@@ -866,6 +982,27 @@ out:
 }
 EXPORT_SYMBOL(mem_cgroup_count_vm_event);
 
+/**
+ * mem_cgroup_zone_lruvec - get the lru list vector for a zone and memcg
+ * @zone: zone of the wanted lruvec
+ * @mem: memcg of the wanted lruvec
+ *
+ * Returns the lru list vector holding pages for the given @zone and
+ * @mem.  This can be the global zone lruvec, if the memory controller
+ * is disabled.
+ */
+struct lruvec *mem_cgroup_zone_lruvec(struct zone *zone,
+                                     struct mem_cgroup *memcg)
+{
+       struct mem_cgroup_per_zone *mz;
+
+       if (mem_cgroup_disabled())
+               return &zone->lruvec;
+
+       mz = mem_cgroup_zoneinfo(memcg, zone_to_nid(zone), zone_idx(zone));
+       return &mz->lruvec;
+}
+
 /*
  * Following LRU functions are allowed to be used without PCG_LOCK.
  * Operations are called by routine of global LRU independently from memcg.
@@ -880,199 +1017,156 @@ EXPORT_SYMBOL(mem_cgroup_count_vm_event);
  * When moving account, the page is not on LRU. It's isolated.
  */
 
-void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
-{
-       struct page_cgroup *pc;
-       struct mem_cgroup_per_zone *mz;
-
-       if (mem_cgroup_disabled())
-               return;
-       pc = lookup_page_cgroup(page);
-       /* can happen while we handle swapcache. */
-       if (!TestClearPageCgroupAcctLRU(pc))
-               return;
-       VM_BUG_ON(!pc->mem_cgroup);
-       /*
-        * We don't check PCG_USED bit. It's cleared when the "page" is finally
-        * removed from global LRU.
-        */
-       mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
-       /* huge page split is done under lru_lock. so, we have no races. */
-       MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       VM_BUG_ON(list_empty(&pc->lru));
-       list_del_init(&pc->lru);
-}
-
-void mem_cgroup_del_lru(struct page *page)
-{
-       mem_cgroup_del_lru_list(page, page_lru(page));
-}
-
-/*
- * Writeback is about to end against a page which has been marked for immediate
- * reclaim.  If it still appears to be reclaimable, move it to the tail of the
- * inactive list.
+/**
+ * mem_cgroup_lru_add_list - account for adding an lru page and return lruvec
+ * @zone: zone of the page
+ * @page: the page
+ * @lru: current lru
+ *
+ * This function accounts for @page being added to @lru, and returns
+ * the lruvec for the given @zone and the memcg @page is charged to.
+ *
+ * The callsite is then responsible for physically linking the page to
+ * the returned lruvec->lists[@lru].
  */
-void mem_cgroup_rotate_reclaimable_page(struct page *page)
+struct lruvec *mem_cgroup_lru_add_list(struct zone *zone, struct page *page,
+                                      enum lru_list lru)
 {
        struct mem_cgroup_per_zone *mz;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc;
-       enum lru_list lru = page_lru(page);
 
        if (mem_cgroup_disabled())
-               return;
+               return &zone->lruvec;
 
        pc = lookup_page_cgroup(page);
-       /* unused or root page is not rotated. */
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
-       list_move_tail(&pc->lru, &mz->lists[lru]);
+       memcg = pc->mem_cgroup;
+       mz = page_cgroup_zoneinfo(memcg, page);
+       /* compound_order() is stabilized through lru_lock */
+       MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
+       return &mz->lruvec;
 }
 
-void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
+/**
+ * mem_cgroup_lru_del_list - account for removing an lru page
+ * @page: the page
+ * @lru: target lru
+ *
+ * This function accounts for @page being removed from @lru.
+ *
+ * The callsite is then responsible for physically unlinking
+ * @page->lru.
+ */
+void mem_cgroup_lru_del_list(struct page *page, enum lru_list lru)
 {
        struct mem_cgroup_per_zone *mz;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc;
 
        if (mem_cgroup_disabled())
                return;
 
        pc = lookup_page_cgroup(page);
-       /* unused or root page is not rotated. */
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
-       list_move(&pc->lru, &mz->lists[lru]);
+       memcg = pc->mem_cgroup;
+       VM_BUG_ON(!memcg);
+       mz = page_cgroup_zoneinfo(memcg, page);
+       /* huge page split is done under lru_lock. so, we have no races. */
+       VM_BUG_ON(MEM_CGROUP_ZSTAT(mz, lru) < (1 << compound_order(page)));
+       MEM_CGROUP_ZSTAT(mz, lru) -= 1 << compound_order(page);
 }
 
-void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
+void mem_cgroup_lru_del(struct page *page)
 {
-       struct page_cgroup *pc;
-       struct mem_cgroup_per_zone *mz;
-
-       if (mem_cgroup_disabled())
-               return;
-       pc = lookup_page_cgroup(page);
-       VM_BUG_ON(PageCgroupAcctLRU(pc));
-       if (!PageCgroupUsed(pc))
-               return;
-       /* Ensure pc->mem_cgroup is visible after reading PCG_USED. */
-       smp_rmb();
-       mz = page_cgroup_zoneinfo(pc->mem_cgroup, page);
-       /* huge page split is done under lru_lock. so, we have no races. */
-       MEM_CGROUP_ZSTAT(mz, lru) += 1 << compound_order(page);
-       SetPageCgroupAcctLRU(pc);
-       if (mem_cgroup_is_root(pc->mem_cgroup))
-               return;
-       list_add(&pc->lru, &mz->lists[lru]);
+       mem_cgroup_lru_del_list(page, page_lru(page));
 }
 
-/*
- * At handling SwapCache and other FUSE stuff, pc->mem_cgroup may be changed
- * while it's linked to lru because the page may be reused after it's fully
- * uncharged. To handle that, unlink page_cgroup from LRU when charge it again.
- * It's done under lock_page and expected that zone->lru_lock isnever held.
+/**
+ * mem_cgroup_lru_move_lists - account for moving a page between lrus
+ * @zone: zone of the page
+ * @page: the page
+ * @from: current lru
+ * @to: target lru
+ *
+ * This function accounts for @page being moved between the lrus @from
+ * and @to, and returns the lruvec for the given @zone and the memcg
+ * @page is charged to.
+ *
+ * The callsite is then responsible for physically relinking
+ * @page->lru to the returned lruvec->lists[@to].
  */
-static void mem_cgroup_lru_del_before_commit(struct page *page)
+struct lruvec *mem_cgroup_lru_move_lists(struct zone *zone,
+                                        struct page *page,
+                                        enum lru_list from,
+                                        enum lru_list to)
 {
-       unsigned long flags;
-       struct zone *zone = page_zone(page);
-       struct page_cgroup *pc = lookup_page_cgroup(page);
-
-       /*
-        * Doing this check without taking ->lru_lock seems wrong but this
-        * is safe. Because if page_cgroup's USED bit is unset, the page
-        * will not be added to any memcg's LRU. If page_cgroup's USED bit is
-        * set, the commit after this will fail, anyway.
-        * This all charge/uncharge is done under some mutual execustion.
-        * So, we don't need to taking care of changes in USED bit.
-        */
-       if (likely(!PageLRU(page)))
-               return;
-
-       spin_lock_irqsave(&zone->lru_lock, flags);
-       /*
-        * Forget old LRU when this page_cgroup is *not* used. This Used bit
-        * is guarded by lock_page() because the page is SwapCache.
-        */
-       if (!PageCgroupUsed(pc))
-               mem_cgroup_del_lru_list(page, page_lru(page));
-       spin_unlock_irqrestore(&zone->lru_lock, flags);
+       /* XXX: Optimize this, especially for @from == @to */
+       mem_cgroup_lru_del_list(page, from);
+       return mem_cgroup_lru_add_list(zone, page, to);
 }
 
-static void mem_cgroup_lru_add_after_commit(struct page *page)
+/*
+ * Checks whether given mem is same or in the root_mem_cgroup's
+ * hierarchy subtree
+ */
+static bool mem_cgroup_same_or_subtree(const struct mem_cgroup *root_memcg,
+               struct mem_cgroup *memcg)
 {
-       unsigned long flags;
-       struct zone *zone = page_zone(page);
-       struct page_cgroup *pc = lookup_page_cgroup(page);
-
-       /* taking care of that the page is added to LRU while we commit it */
-       if (likely(!PageLRU(page)))
-               return;
-       spin_lock_irqsave(&zone->lru_lock, flags);
-       /* link when the page is linked to LRU but page_cgroup isn't */
-       if (PageLRU(page) && !PageCgroupAcctLRU(pc))
-               mem_cgroup_add_lru_list(page, page_lru(page));
-       spin_unlock_irqrestore(&zone->lru_lock, flags);
-}
-
+       if (root_memcg != memcg) {
+               return (root_memcg->use_hierarchy &&
+                       css_is_ancestor(&memcg->css, &root_memcg->css));
+       }
 
-void mem_cgroup_move_lists(struct page *page,
-                          enum lru_list from, enum lru_list to)
-{
-       if (mem_cgroup_disabled())
-               return;
-       mem_cgroup_del_lru_list(page, from);
-       mem_cgroup_add_lru_list(page, to);
+       return true;
 }
 
-int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
+int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *memcg)
 {
        int ret;
        struct mem_cgroup *curr = NULL;
        struct task_struct *p;
 
        p = find_lock_task_mm(task);
-       if (!p)
-               return 0;
-       curr = try_get_mem_cgroup_from_mm(p->mm);
-       task_unlock(p);
+       if (p) {
+               curr = try_get_mem_cgroup_from_mm(p->mm);
+               task_unlock(p);
+       } else {
+               /*
+                * All threads may have already detached their mm's, but the oom
+                * killer still needs to detect if they have already been oom
+                * killed to prevent needlessly killing additional tasks.
+                */
+               task_lock(task);
+               curr = mem_cgroup_from_task(task);
+               if (curr)
+                       css_get(&curr->css);
+               task_unlock(task);
+       }
        if (!curr)
                return 0;
        /*
-        * We should check use_hierarchy of "mem" not "curr". Because checking
+        * We should check use_hierarchy of "memcg" not "curr". Because checking
         * use_hierarchy of "curr" here make this function true if hierarchy is
-        * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
-        * hierarchy(even if use_hierarchy is disabled in "mem").
+        * enabled in "curr" and "curr" is a child of "memcg" in *cgroup*
+        * hierarchy(even if use_hierarchy is disabled in "memcg").
         */
-       if (mem->use_hierarchy)
-               ret = css_is_ancestor(&curr->css, &mem->css);
-       else
-               ret = (curr == mem);
+       ret = mem_cgroup_same_or_subtree(memcg, curr);
        css_put(&curr->css);
        return ret;
 }
 
-static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
+int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
 {
-       unsigned long active;
+       unsigned long inactive_ratio;
+       int nid = zone_to_nid(zone);
+       int zid = zone_idx(zone);
        unsigned long inactive;
+       unsigned long active;
        unsigned long gb;
-       unsigned long inactive_ratio;
 
-       inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
-       active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
+       inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                               BIT(LRU_INACTIVE_ANON));
+       active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                             BIT(LRU_ACTIVE_ANON));
 
        gb = (inactive + active) >> (30 - PAGE_SHIFT);
        if (gb)
@@ -1080,140 +1174,23 @@ static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_
        else
                inactive_ratio = 1;
 
-       if (present_pages) {
-               present_pages[0] = inactive;
-               present_pages[1] = active;
-       }
-
-       return inactive_ratio;
-}
-
-int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
-{
-       unsigned long active;
-       unsigned long inactive;
-       unsigned long present_pages[2];
-       unsigned long inactive_ratio;
-
-       inactive_ratio = calc_inactive_ratio(memcg, present_pages);
-
-       inactive = present_pages[0];
-       active = present_pages[1];
-
-       if (inactive * inactive_ratio < active)
-               return 1;
-
-       return 0;
+       return inactive * inactive_ratio < active;
 }
 
-int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
+int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg, struct zone *zone)
 {
        unsigned long active;
        unsigned long inactive;
-
-       inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
-       active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
-
-       return (active > inactive);
-}
-
-unsigned long mem_cgroup_zone_nr_lru_pages(struct mem_cgroup *memcg,
-                                               struct zone *zone,
-                                               enum lru_list lru)
-{
-       int nid = zone_to_nid(zone);
        int zid = zone_idx(zone);
-       struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
-
-       return MEM_CGROUP_ZSTAT(mz, lru);
-}
-
-#ifdef CONFIG_NUMA
-static unsigned long mem_cgroup_node_nr_file_lru_pages(struct mem_cgroup *memcg,
-                                                       int nid)
-{
-       unsigned long ret;
-
-       ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_FILE) +
-               mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_FILE);
-
-       return ret;
-}
-
-static unsigned long mem_cgroup_nr_file_lru_pages(struct mem_cgroup *memcg)
-{
-       u64 total = 0;
-       int nid;
-
-       for_each_node_state(nid, N_HIGH_MEMORY)
-               total += mem_cgroup_node_nr_file_lru_pages(memcg, nid);
-
-       return total;
-}
-
-static unsigned long mem_cgroup_node_nr_anon_lru_pages(struct mem_cgroup *memcg,
-                                                       int nid)
-{
-       unsigned long ret;
-
-       ret = mem_cgroup_get_zonestat_node(memcg, nid, LRU_INACTIVE_ANON) +
-               mem_cgroup_get_zonestat_node(memcg, nid, LRU_ACTIVE_ANON);
-
-       return ret;
-}
-
-static unsigned long mem_cgroup_nr_anon_lru_pages(struct mem_cgroup *memcg)
-{
-       u64 total = 0;
-       int nid;
-
-       for_each_node_state(nid, N_HIGH_MEMORY)
-               total += mem_cgroup_node_nr_anon_lru_pages(memcg, nid);
-
-       return total;
-}
-
-static unsigned long
-mem_cgroup_node_nr_unevictable_lru_pages(struct mem_cgroup *memcg, int nid)
-{
-       return mem_cgroup_get_zonestat_node(memcg, nid, LRU_UNEVICTABLE);
-}
-
-static unsigned long
-mem_cgroup_nr_unevictable_lru_pages(struct mem_cgroup *memcg)
-{
-       u64 total = 0;
-       int nid;
-
-       for_each_node_state(nid, N_HIGH_MEMORY)
-               total += mem_cgroup_node_nr_unevictable_lru_pages(memcg, nid);
-
-       return total;
-}
-
-static unsigned long mem_cgroup_node_nr_lru_pages(struct mem_cgroup *memcg,
-                                                       int nid)
-{
-       enum lru_list l;
-       u64 total = 0;
-
-       for_each_lru(l)
-               total += mem_cgroup_get_zonestat_node(memcg, nid, l);
-
-       return total;
-}
-
-static unsigned long mem_cgroup_nr_lru_pages(struct mem_cgroup *memcg)
-{
-       u64 total = 0;
-       int nid;
+       int nid = zone_to_nid(zone);
 
-       for_each_node_state(nid, N_HIGH_MEMORY)
-               total += mem_cgroup_node_nr_lru_pages(memcg, nid);
+       inactive = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                               BIT(LRU_INACTIVE_FILE));
+       active = mem_cgroup_zone_nr_lru_pages(memcg, nid, zid,
+                                             BIT(LRU_ACTIVE_FILE));
 
-       return total;
+       return (active > inactive);
 }
-#endif /* CONFIG_NUMA */
 
 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
                                                      struct zone *zone)
@@ -1243,67 +1220,6 @@ mem_cgroup_get_reclaim_stat_from_page(struct page *page)
        return &mz->reclaim_stat;
 }
 
-unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
-                                       struct list_head *dst,
-                                       unsigned long *scanned, int order,
-                                       int mode, struct zone *z,
-                                       struct mem_cgroup *mem_cont,
-                                       int active, int file)
-{
-       unsigned long nr_taken = 0;
-       struct page *page;
-       unsigned long scan;
-       LIST_HEAD(pc_list);
-       struct list_head *src;
-       struct page_cgroup *pc, *tmp;
-       int nid = zone_to_nid(z);
-       int zid = zone_idx(z);
-       struct mem_cgroup_per_zone *mz;
-       int lru = LRU_FILE * file + active;
-       int ret;
-
-       BUG_ON(!mem_cont);
-       mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
-       src = &mz->lists[lru];
-
-       scan = 0;
-       list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
-               if (scan >= nr_to_scan)
-                       break;
-
-               if (unlikely(!PageCgroupUsed(pc)))
-                       continue;
-
-               page = lookup_cgroup_page(pc);
-
-               if (unlikely(!PageLRU(page)))
-                       continue;
-
-               scan++;
-               ret = __isolate_lru_page(page, mode, file);
-               switch (ret) {
-               case 0:
-                       list_move(&page->lru, dst);
-                       mem_cgroup_del_lru(page);
-                       nr_taken += hpage_nr_pages(page);
-                       break;
-               case -EBUSY:
-                       /* we don't affect global LRU but rotate in our LRU */
-                       mem_cgroup_rotate_lru_list(page, page_lru(page));
-                       break;
-               default:
-                       break;
-               }
-       }
-
-       *scanned = scan;
-
-       trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
-                                     0, 0, 0, mode);
-
-       return nr_taken;
-}
-
 #define mem_cgroup_from_res_counter(counter, member)   \
        container_of(counter, struct mem_cgroup, member)
 
@@ -1314,17 +1230,17 @@ unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
  * Returns the maximum amount of memory @mem can be charged with, in
  * pages.
  */
-static unsigned long mem_cgroup_margin(struct mem_cgroup *mem)
+static unsigned long mem_cgroup_margin(struct mem_cgroup *memcg)
 {
        unsigned long long margin;
 
-       margin = res_counter_margin(&mem->res);
+       margin = res_counter_margin(&memcg->res);
        if (do_swap_account)
-               margin = min(margin, res_counter_margin(&mem->memsw));
+               margin = min(margin, res_counter_margin(&memcg->memsw));
        return margin >> PAGE_SHIFT;
 }
 
-static unsigned int get_swappiness(struct mem_cgroup *memcg)
+int mem_cgroup_swappiness(struct mem_cgroup *memcg)
 {
        struct cgroup *cgrp = memcg->css.cgroup;
 
@@ -1335,33 +1251,33 @@ static unsigned int get_swappiness(struct mem_cgroup *memcg)
        return memcg->swappiness;
 }
 
-static void mem_cgroup_start_move(struct mem_cgroup *mem)
+static void mem_cgroup_start_move(struct mem_cgroup *memcg)
 {
        int cpu;
 
        get_online_cpus();
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for_each_online_cpu(cpu)
-               per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
-       mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
-       spin_unlock(&mem->pcp_counter_lock);
+               per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
+       memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
+       spin_unlock(&memcg->pcp_counter_lock);
        put_online_cpus();
 
        synchronize_rcu();
 }
 
-static void mem_cgroup_end_move(struct mem_cgroup *mem)
+static void mem_cgroup_end_move(struct mem_cgroup *memcg)
 {
        int cpu;
 
-       if (!mem)
+       if (!memcg)
                return;
        get_online_cpus();
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for_each_online_cpu(cpu)
-               per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
-       mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
-       spin_unlock(&mem->pcp_counter_lock);
+               per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
+       memcg->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
+       spin_unlock(&memcg->pcp_counter_lock);
        put_online_cpus();
 }
 /*
@@ -1376,13 +1292,13 @@ static void mem_cgroup_end_move(struct mem_cgroup *mem)
  *                       waiting at hith-memory prressure caused by "move".
  */
 
-static bool mem_cgroup_stealed(struct mem_cgroup *mem)
+static bool mem_cgroup_stealed(struct mem_cgroup *memcg)
 {
        VM_BUG_ON(!rcu_read_lock_held());
-       return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
+       return this_cpu_read(memcg->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
 }
 
-static bool mem_cgroup_under_move(struct mem_cgroup *mem)
+static bool mem_cgroup_under_move(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *from;
        struct mem_cgroup *to;
@@ -1396,19 +1312,18 @@ static bool mem_cgroup_under_move(struct mem_cgroup *mem)
        to = mc.to;
        if (!from)
                goto unlock;
-       if (from == mem || to == mem
-           || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
-           || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
-               ret = true;
+
+       ret = mem_cgroup_same_or_subtree(memcg, from)
+               || mem_cgroup_same_or_subtree(memcg, to);
 unlock:
        spin_unlock(&mc.lock);
        return ret;
 }
 
-static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
+static bool mem_cgroup_wait_acct_move(struct mem_cgroup *memcg)
 {
        if (mc.moving_task && current != mc.moving_task) {
-               if (mem_cgroup_under_move(mem)) {
+               if (mem_cgroup_under_move(memcg)) {
                        DEFINE_WAIT(wait);
                        prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
                        /* moving charge context might have finished. */
@@ -1492,12 +1407,12 @@ done:
  * This function returns the number of memcg under hierarchy tree. Returns
  * 1(self count) if no children.
  */
-static int mem_cgroup_count_children(struct mem_cgroup *mem)
+static int mem_cgroup_count_children(struct mem_cgroup *memcg)
 {
        int num = 0;
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                num++;
        return num;
 }
@@ -1521,43 +1436,64 @@ u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
        return min(limit, memsw);
 }
 
-/*
- * Visit the first child (need not be the first child as per the ordering
- * of the cgroup list, since we track last_scanned_child) of @mem and use
- * that to reclaim free pages from.
- */
-static struct mem_cgroup *
-mem_cgroup_select_victim(struct mem_cgroup *root_mem)
+static unsigned long mem_cgroup_reclaim(struct mem_cgroup *memcg,
+                                       gfp_t gfp_mask,
+                                       unsigned long flags)
 {
-       struct mem_cgroup *ret = NULL;
-       struct cgroup_subsys_state *css;
-       int nextid, found;
-
-       if (!root_mem->use_hierarchy) {
-               css_get(&root_mem->css);
-               ret = root_mem;
-       }
+       unsigned long total = 0;
+       bool noswap = false;
+       int loop;
 
-       while (!ret) {
-               rcu_read_lock();
-               nextid = root_mem->last_scanned_child + 1;
-               css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
-                                  &found);
-               if (css && css_tryget(css))
-                       ret = container_of(css, struct mem_cgroup, css);
+       if (flags & MEM_CGROUP_RECLAIM_NOSWAP)
+               noswap = true;
+       if (!(flags & MEM_CGROUP_RECLAIM_SHRINK) && memcg->memsw_is_minimum)
+               noswap = true;
 
-               rcu_read_unlock();
-               /* Updates scanning parameter */
-               if (!css) {
-                       /* this means start scan from ID:1 */
-                       root_mem->last_scanned_child = 0;
-               } else
-                       root_mem->last_scanned_child = found;
+       for (loop = 0; loop < MEM_CGROUP_MAX_RECLAIM_LOOPS; loop++) {
+               if (loop)
+                       drain_all_stock_async(memcg);
+               total += try_to_free_mem_cgroup_pages(memcg, gfp_mask, noswap);
+               /*
+                * Allow limit shrinkers, which are triggered directly
+                * by userspace, to catch signals and stop reclaim
+                * after minimal progress, regardless of the margin.
+                */
+               if (total && (flags & MEM_CGROUP_RECLAIM_SHRINK))
+                       break;
+               if (mem_cgroup_margin(memcg))
+                       break;
+               /*
+                * If nothing was reclaimed after two attempts, there
+                * may be no reclaimable pages in this hierarchy.
+                */
+               if (loop && !total)
+                       break;
        }
-
-       return ret;
+       return total;
 }
 
+/**
+ * test_mem_cgroup_node_reclaimable
+ * @mem: the target memcg
+ * @nid: the node ID to be checked.
+ * @noswap : specify true here if the user wants flle only information.
+ *
+ * This function returns whether the specified memcg contains any
+ * reclaimable pages on a node. Returns true if there are any reclaimable
+ * pages in the node.
+ */
+static bool test_mem_cgroup_node_reclaimable(struct mem_cgroup *memcg,
+               int nid, bool noswap)
+{
+       if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_FILE))
+               return true;
+       if (noswap || !total_swap_pages)
+               return false;
+       if (mem_cgroup_node_nr_lru_pages(memcg, nid, LRU_ALL_ANON))
+               return true;
+       return false;
+
+}
 #if MAX_NUMNODES > 1
 
 /*
@@ -1566,29 +1502,29 @@ mem_cgroup_select_victim(struct mem_cgroup *root_mem)
  * nodes based on the zonelist. So update the list loosely once per 10 secs.
  *
  */
-static void mem_cgroup_may_update_nodemask(struct mem_cgroup *mem)
+static void mem_cgroup_may_update_nodemask(struct mem_cgroup *memcg)
 {
        int nid;
-
-       if (time_after(mem->next_scan_node_update, jiffies))
+       /*
+        * numainfo_events > 0 means there was at least NUMAINFO_EVENTS_TARGET
+        * pagein/pageout changes since the last update.
+        */
+       if (!atomic_read(&memcg->numainfo_events))
+               return;
+       if (atomic_inc_return(&memcg->numainfo_updating) > 1)
                return;
 
-       mem->next_scan_node_update = jiffies + 10*HZ;
        /* make a nodemask where this memcg uses memory from */
-       mem->scan_nodes = node_states[N_HIGH_MEMORY];
+       memcg->scan_nodes = node_states[N_HIGH_MEMORY];
 
        for_each_node_mask(nid, node_states[N_HIGH_MEMORY]) {
 
-               if (mem_cgroup_get_zonestat_node(mem, nid, LRU_INACTIVE_FILE) ||
-                   mem_cgroup_get_zonestat_node(mem, nid, LRU_ACTIVE_FILE))
-                       continue;
-
-               if (total_swap_pages &&
-                   (mem_cgroup_get_zonestat_node(mem, nid, LRU_INACTIVE_ANON) ||
-                    mem_cgroup_get_zonestat_node(mem, nid, LRU_ACTIVE_ANON)))
-                       continue;
-               node_clear(nid, mem->scan_nodes);
+               if (!test_mem_cgroup_node_reclaimable(memcg, nid, false))
+                       node_clear(nid, memcg->scan_nodes);
        }
+
+       atomic_set(&memcg->numainfo_events, 0);
+       atomic_set(&memcg->numainfo_updating, 0);
 }
 
 /*
@@ -1603,16 +1539,16 @@ static void mem_cgroup_may_update_nodemask(struct mem_cgroup *mem)
  *
  * Now, we use round-robin. Better algorithm is welcomed.
  */
-int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
+int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
 {
        int node;
 
-       mem_cgroup_may_update_nodemask(mem);
-       node = mem->last_scanned_node;
+       mem_cgroup_may_update_nodemask(memcg);
+       node = memcg->last_scanned_node;
 
-       node = next_node(node, mem->scan_nodes);
+       node = next_node(node, memcg->scan_nodes);
        if (node == MAX_NUMNODES)
-               node = first_node(mem->scan_nodes);
+               node = first_node(memcg->scan_nodes);
        /*
         * We call this when we hit limit, not when pages are added to LRU.
         * No LRU may hold pages because all pages are UNEVICTABLE or
@@ -1622,66 +1558,86 @@ int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
        if (unlikely(node == MAX_NUMNODES))
                node = numa_node_id();
 
-       mem->last_scanned_node = node;
+       memcg->last_scanned_node = node;
        return node;
 }
 
+/*
+ * Check all nodes whether it contains reclaimable pages or not.
+ * For quick scan, we make use of scan_nodes. This will allow us to skip
+ * unused nodes. But scan_nodes is lazily updated and may not cotain
+ * enough new information. We need to do double check.
+ */
+bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
+{
+       int nid;
+
+       /*
+        * quick check...making use of scan_node.
+        * We can skip unused nodes.
+        */
+       if (!nodes_empty(memcg->scan_nodes)) {
+               for (nid = first_node(memcg->scan_nodes);
+                    nid < MAX_NUMNODES;
+                    nid = next_node(nid, memcg->scan_nodes)) {
+
+                       if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
+                               return true;
+               }
+       }
+       /*
+        * Check rest of nodes.
+        */
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               if (node_isset(nid, memcg->scan_nodes))
+                       continue;
+               if (test_mem_cgroup_node_reclaimable(memcg, nid, noswap))
+                       return true;
+       }
+       return false;
+}
+
 #else
-int mem_cgroup_select_victim_node(struct mem_cgroup *mem)
+int mem_cgroup_select_victim_node(struct mem_cgroup *memcg)
 {
        return 0;
 }
+
+bool mem_cgroup_reclaimable(struct mem_cgroup *memcg, bool noswap)
+{
+       return test_mem_cgroup_node_reclaimable(memcg, 0, noswap);
+}
 #endif
 
-/*
- * Scan the hierarchy if needed to reclaim memory. We remember the last child
- * we reclaimed from, so that we don't end up penalizing one child extensively
- * based on its position in the children list.
- *
- * root_mem is the original ancestor that we've been reclaim from.
- *
- * We give up and return to the caller when we visit root_mem twice.
- * (other groups can be removed while we're walking....)
- *
- * If shrink==true, for avoiding to free too much, this returns immedieately.
- */
-static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
-                                               struct zone *zone,
-                                               gfp_t gfp_mask,
-                                               unsigned long reclaim_options,
-                                               unsigned long *total_scanned)
-{
-       struct mem_cgroup *victim;
-       int ret, total = 0;
+static int mem_cgroup_soft_reclaim(struct mem_cgroup *root_memcg,
+                                  struct zone *zone,
+                                  gfp_t gfp_mask,
+                                  unsigned long *total_scanned)
+{
+       struct mem_cgroup *victim = NULL;
+       int total = 0;
        int loop = 0;
-       bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
-       bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
-       bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
        unsigned long excess;
        unsigned long nr_scanned;
+       struct mem_cgroup_reclaim_cookie reclaim = {
+               .zone = zone,
+               .priority = 0,
+       };
 
-       excess = res_counter_soft_limit_excess(&root_mem->res) >> PAGE_SHIFT;
-
-       /* If memsw_is_minimum==1, swap-out is of-no-use. */
-       if (root_mem->memsw_is_minimum)
-               noswap = true;
+       excess = res_counter_soft_limit_excess(&root_memcg->res) >> PAGE_SHIFT;
 
        while (1) {
-               victim = mem_cgroup_select_victim(root_mem);
-               if (victim == root_mem) {
+               victim = mem_cgroup_iter(root_memcg, victim, &reclaim);
+               if (!victim) {
                        loop++;
-                       if (loop >= 1)
-                               drain_all_stock_async();
                        if (loop >= 2) {
                                /*
                                 * If we have not been able to reclaim
                                 * anything, it might because there are
                                 * no reclaimable pages under this hierarchy
                                 */
-                               if (!check_soft || !total) {
-                                       css_put(&victim->css);
+                               if (!total)
                                        break;
-                               }
                                /*
                                 * We want to do more targeted reclaim.
                                 * excess >> 2 is not to excessive so as to
@@ -1689,64 +1645,83 @@ static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
                                 * coming back to reclaim from this cgroup
                                 */
                                if (total >= (excess >> 2) ||
-                                       (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
-                                       css_put(&victim->css);
+                                       (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS))
                                        break;
-                               }
                        }
-               }
-               if (!mem_cgroup_local_usage(victim)) {
-                       /* this cgroup's local usage == 0 */
-                       css_put(&victim->css);
                        continue;
                }
-               /* we use swappiness of local cgroup */
-               if (check_soft) {
-                       ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
-                               noswap, get_swappiness(victim), zone,
-                               &nr_scanned);
-                       *total_scanned += nr_scanned;
-               } else
-                       ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
-                                               noswap, get_swappiness(victim));
-               css_put(&victim->css);
-               /*
-                * At shrinking usage, we can't check we should stop here or
-                * reclaim more. It's depends on callers. last_scanned_child
-                * will work enough for keeping fairness under tree.
-                */
-               if (shrink)
-                       return ret;
-               total += ret;
-               if (check_soft) {
-                       if (!res_counter_soft_limit_excess(&root_mem->res))
-                               return total;
-               } else if (mem_cgroup_margin(root_mem))
-                       return total;
+               if (!mem_cgroup_reclaimable(victim, false))
+                       continue;
+               total += mem_cgroup_shrink_node_zone(victim, gfp_mask, false,
+                                                    zone, &nr_scanned);
+               *total_scanned += nr_scanned;
+               if (!res_counter_soft_limit_excess(&root_memcg->res))
+                       break;
        }
+       mem_cgroup_iter_break(root_memcg, victim);
        return total;
 }
 
 /*
  * Check OOM-Killer is already running under our hierarchy.
  * If someone is running, return false.
+ * Has to be called with memcg_oom_lock
  */
-static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
+static bool mem_cgroup_oom_lock(struct mem_cgroup *memcg)
 {
-       int x, lock_count = 0;
-       struct mem_cgroup *iter;
+       struct mem_cgroup *iter, *failed = NULL;
 
-       for_each_mem_cgroup_tree(iter, mem) {
-               x = atomic_inc_return(&iter->oom_lock);
-               lock_count = max(x, lock_count);
+       for_each_mem_cgroup_tree(iter, memcg) {
+               if (iter->oom_lock) {
+                       /*
+                        * this subtree of our hierarchy is already locked
+                        * so we cannot give a lock.
+                        */
+                       failed = iter;
+                       mem_cgroup_iter_break(memcg, iter);
+                       break;
+               } else
+                       iter->oom_lock = true;
        }
 
-       if (lock_count == 1)
+       if (!failed)
                return true;
+
+       /*
+        * OK, we failed to lock the whole subtree so we have to clean up
+        * what we set up to the failing subtree
+        */
+       for_each_mem_cgroup_tree(iter, memcg) {
+               if (iter == failed) {
+                       mem_cgroup_iter_break(memcg, iter);
+                       break;
+               }
+               iter->oom_lock = false;
+       }
        return false;
 }
 
-static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
+/*
+ * Has to be called with memcg_oom_lock
+ */
+static int mem_cgroup_oom_unlock(struct mem_cgroup *memcg)
+{
+       struct mem_cgroup *iter;
+
+       for_each_mem_cgroup_tree(iter, memcg)
+               iter->oom_lock = false;
+       return 0;
+}
+
+static void mem_cgroup_mark_under_oom(struct mem_cgroup *memcg)
+{
+       struct mem_cgroup *iter;
+
+       for_each_mem_cgroup_tree(iter, memcg)
+               atomic_inc(&iter->under_oom);
+}
+
+static void mem_cgroup_unmark_under_oom(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *iter;
 
@@ -1755,13 +1730,11 @@ static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
         * mem_cgroup_oom_lock() may not be called. We have to use
         * atomic_add_unless() here.
         */
-       for_each_mem_cgroup_tree(iter, mem)
-               atomic_add_unless(&iter->oom_lock, -1, 0);
-       return 0;
+       for_each_mem_cgroup_tree(iter, memcg)
+               atomic_add_unless(&iter->under_oom, -1, 0);
 }
 
-
-static DEFINE_MUTEX(memcg_oom_mutex);
+static DEFINE_SPINLOCK(memcg_oom_lock);
 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
 
 struct oom_wait_info {
@@ -1772,85 +1745,85 @@ struct oom_wait_info {
 static int memcg_oom_wake_function(wait_queue_t *wait,
        unsigned mode, int sync, void *arg)
 {
-       struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
+       struct mem_cgroup *wake_memcg = (struct mem_cgroup *)arg,
+                         *oom_wait_memcg;
        struct oom_wait_info *oom_wait_info;
 
        oom_wait_info = container_of(wait, struct oom_wait_info, wait);
+       oom_wait_memcg = oom_wait_info->mem;
 
-       if (oom_wait_info->mem == wake_mem)
-               goto wakeup;
-       /* if no hierarchy, no match */
-       if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
-               return 0;
        /*
         * Both of oom_wait_info->mem and wake_mem are stable under us.
         * Then we can use css_is_ancestor without taking care of RCU.
         */
-       if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
-           !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
+       if (!mem_cgroup_same_or_subtree(oom_wait_memcg, wake_memcg)
+               && !mem_cgroup_same_or_subtree(wake_memcg, oom_wait_memcg))
                return 0;
-
-wakeup:
        return autoremove_wake_function(wait, mode, sync, arg);
 }
 
-static void memcg_wakeup_oom(struct mem_cgroup *mem)
+static void memcg_wakeup_oom(struct mem_cgroup *memcg)
 {
-       /* for filtering, pass "mem" as argument. */
-       __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
+       /* for filtering, pass "memcg" as argument. */
+       __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, memcg);
 }
 
-static void memcg_oom_recover(struct mem_cgroup *mem)
+static void memcg_oom_recover(struct mem_cgroup *memcg)
 {
-       if (mem && atomic_read(&mem->oom_lock))
-               memcg_wakeup_oom(mem);
+       if (memcg && atomic_read(&memcg->under_oom))
+               memcg_wakeup_oom(memcg);
 }
 
 /*
  * try to call OOM killer. returns false if we should exit memory-reclaim loop.
  */
-bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
+bool mem_cgroup_handle_oom(struct mem_cgroup *memcg, gfp_t mask)
 {
        struct oom_wait_info owait;
        bool locked, need_to_kill;
 
-       owait.mem = mem;
+       owait.mem = memcg;
        owait.wait.flags = 0;
        owait.wait.func = memcg_oom_wake_function;
        owait.wait.private = current;
        INIT_LIST_HEAD(&owait.wait.task_list);
        need_to_kill = true;
-       /* At first, try to OOM lock hierarchy under mem.*/
-       mutex_lock(&memcg_oom_mutex);
-       locked = mem_cgroup_oom_lock(mem);
+       mem_cgroup_mark_under_oom(memcg);
+
+       /* At first, try to OOM lock hierarchy under memcg.*/
+       spin_lock(&memcg_oom_lock);
+       locked = mem_cgroup_oom_lock(memcg);
        /*
         * Even if signal_pending(), we can't quit charge() loop without
         * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
         * under OOM is always welcomed, use TASK_KILLABLE here.
         */
        prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
-       if (!locked || mem->oom_kill_disable)
+       if (!locked || memcg->oom_kill_disable)
                need_to_kill = false;
        if (locked)
-               mem_cgroup_oom_notify(mem);
-       mutex_unlock(&memcg_oom_mutex);
+               mem_cgroup_oom_notify(memcg);
+       spin_unlock(&memcg_oom_lock);
 
        if (need_to_kill) {
                finish_wait(&memcg_oom_waitq, &owait.wait);
-               mem_cgroup_out_of_memory(mem, mask);
+               mem_cgroup_out_of_memory(memcg, mask);
        } else {
                schedule();
                finish_wait(&memcg_oom_waitq, &owait.wait);
        }
-       mutex_lock(&memcg_oom_mutex);
-       mem_cgroup_oom_unlock(mem);
-       memcg_wakeup_oom(mem);
-       mutex_unlock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
+       if (locked)
+               mem_cgroup_oom_unlock(memcg);
+       memcg_wakeup_oom(memcg);
+       spin_unlock(&memcg_oom_lock);
+
+       mem_cgroup_unmark_under_oom(memcg);
 
        if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
                return false;
        /* Give chance to dying process */
-       schedule_timeout(1);
+       schedule_timeout_uninterruptible(1);
        return true;
 }
 
@@ -1881,25 +1854,25 @@ bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
 void mem_cgroup_update_page_stat(struct page *page,
                                 enum mem_cgroup_page_stat_item idx, int val)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        struct page_cgroup *pc = lookup_page_cgroup(page);
        bool need_unlock = false;
        unsigned long uninitialized_var(flags);
 
-       if (unlikely(!pc))
+       if (mem_cgroup_disabled())
                return;
 
        rcu_read_lock();
-       mem = pc->mem_cgroup;
-       if (unlikely(!mem || !PageCgroupUsed(pc)))
+       memcg = pc->mem_cgroup;
+       if (unlikely(!memcg || !PageCgroupUsed(pc)))
                goto out;
        /* pc->mem_cgroup is unstable ? */
-       if (unlikely(mem_cgroup_stealed(mem)) || PageTransHuge(page)) {
+       if (unlikely(mem_cgroup_stealed(memcg)) || PageTransHuge(page)) {
                /* take a lock against to access pc->mem_cgroup */
                move_lock_page_cgroup(pc, &flags);
                need_unlock = true;
-               mem = pc->mem_cgroup;
-               if (!mem || !PageCgroupUsed(pc))
+               memcg = pc->mem_cgroup;
+               if (!memcg || !PageCgroupUsed(pc))
                        goto out;
        }
 
@@ -1915,7 +1888,7 @@ void mem_cgroup_update_page_stat(struct page *page,
                BUG();
        }
 
-       this_cpu_add(mem->stat->count[idx], val);
+       this_cpu_add(memcg->stat->count[idx], val);
 
 out:
        if (unlikely(need_unlock))
@@ -1934,9 +1907,11 @@ struct memcg_stock_pcp {
        struct mem_cgroup *cached; /* this never be root cgroup */
        unsigned int nr_pages;
        struct work_struct work;
+       unsigned long flags;
+#define FLUSHING_CACHED_CHARGE (0)
 };
 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
-static atomic_t memcg_drain_count;
+static DEFINE_MUTEX(percpu_charge_mutex);
 
 /*
  * Try to consume stocked charge on this cpu. If success, one page is consumed
@@ -1944,13 +1919,13 @@ static atomic_t memcg_drain_count;
  * cgroup which is not current target, returns false. This stock will be
  * refilled.
  */
-static bool consume_stock(struct mem_cgroup *mem)
+static bool consume_stock(struct mem_cgroup *memcg)
 {
        struct memcg_stock_pcp *stock;
        bool ret = true;
 
        stock = &get_cpu_var(memcg_stock);
-       if (mem == stock->cached && stock->nr_pages)
+       if (memcg == stock->cached && stock->nr_pages)
                stock->nr_pages--;
        else /* need to call res_counter_charge */
                ret = false;
@@ -1984,95 +1959,126 @@ static void drain_local_stock(struct work_struct *dummy)
 {
        struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
        drain_stock(stock);
+       clear_bit(FLUSHING_CACHED_CHARGE, &stock->flags);
 }
 
 /*
  * Cache charges(val) which is from res_counter, to local per_cpu area.
  * This will be consumed by consume_stock() function, later.
  */
-static void refill_stock(struct mem_cgroup *mem, unsigned int nr_pages)
+static void refill_stock(struct mem_cgroup *memcg, unsigned int nr_pages)
 {
        struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
 
-       if (stock->cached != mem) { /* reset if necessary */
+       if (stock->cached != memcg) { /* reset if necessary */
                drain_stock(stock);
-               stock->cached = mem;
+               stock->cached = memcg;
        }
        stock->nr_pages += nr_pages;
        put_cpu_var(memcg_stock);
 }
 
 /*
- * Tries to drain stocked charges in other cpus. This function is asynchronous
- * and just put a work per cpu for draining localy on each cpu. Caller can
- * expects some charges will be back to res_counter later but cannot wait for
- * it.
+ * Drains all per-CPU charge caches for given root_memcg resp. subtree
+ * of the hierarchy under it. sync flag says whether we should block
+ * until the work is done.
  */
-static void drain_all_stock_async(void)
+static void drain_all_stock(struct mem_cgroup *root_memcg, bool sync)
 {
-       int cpu;
-       /* This function is for scheduling "drain" in asynchronous way.
-        * The result of "drain" is not directly handled by callers. Then,
-        * if someone is calling drain, we don't have to call drain more.
-        * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
-        * there is a race. We just do loose check here.
-        */
-       if (atomic_read(&memcg_drain_count))
-               return;
+       int cpu, curcpu;
+
        /* Notify other cpus that system-wide "drain" is running */
-       atomic_inc(&memcg_drain_count);
        get_online_cpus();
+       curcpu = get_cpu();
        for_each_online_cpu(cpu) {
                struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
-               schedule_work_on(cpu, &stock->work);
+               struct mem_cgroup *memcg;
+
+               memcg = stock->cached;
+               if (!memcg || !stock->nr_pages)
+                       continue;
+               if (!mem_cgroup_same_or_subtree(root_memcg, memcg))
+                       continue;
+               if (!test_and_set_bit(FLUSHING_CACHED_CHARGE, &stock->flags)) {
+                       if (cpu == curcpu)
+                               drain_local_stock(&stock->work);
+                       else
+                               schedule_work_on(cpu, &stock->work);
+               }
        }
+       put_cpu();
+
+       if (!sync)
+               goto out;
+
+       for_each_online_cpu(cpu) {
+               struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
+               if (test_bit(FLUSHING_CACHED_CHARGE, &stock->flags))
+                       flush_work(&stock->work);
+       }
+out:
        put_online_cpus();
-       atomic_dec(&memcg_drain_count);
-       /* We don't wait for flush_work */
+}
+
+/*
+ * Tries to drain stocked charges in other cpus. This function is asynchronous
+ * and just put a work per cpu for draining localy on each cpu. Caller can
+ * expects some charges will be back to res_counter later but cannot wait for
+ * it.
+ */
+static void drain_all_stock_async(struct mem_cgroup *root_memcg)
+{
+       /*
+        * If someone calls draining, avoid adding more kworker runs.
+        */
+       if (!mutex_trylock(&percpu_charge_mutex))
+               return;
+       drain_all_stock(root_memcg, false);
+       mutex_unlock(&percpu_charge_mutex);
 }
 
 /* This is a synchronous drain interface. */
-static void drain_all_stock_sync(void)
+static void drain_all_stock_sync(struct mem_cgroup *root_memcg)
 {
        /* called when force_empty is called */
-       atomic_inc(&memcg_drain_count);
-       schedule_on_each_cpu(drain_local_stock);
-       atomic_dec(&memcg_drain_count);
+       mutex_lock(&percpu_charge_mutex);
+       drain_all_stock(root_memcg, true);
+       mutex_unlock(&percpu_charge_mutex);
 }
 
 /*
  * This function drains percpu counter value from DEAD cpu and
  * move it to local cpu. Note that this function can be preempted.
  */
-static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
+static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *memcg, int cpu)
 {
        int i;
 
-       spin_lock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
        for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
-               long x = per_cpu(mem->stat->count[i], cpu);
+               long x = per_cpu(memcg->stat->count[i], cpu);
 
-               per_cpu(mem->stat->count[i], cpu) = 0;
-               mem->nocpu_base.count[i] += x;
+               per_cpu(memcg->stat->count[i], cpu) = 0;
+               memcg->nocpu_base.count[i] += x;
        }
        for (i = 0; i < MEM_CGROUP_EVENTS_NSTATS; i++) {
-               unsigned long x = per_cpu(mem->stat->events[i], cpu);
+               unsigned long x = per_cpu(memcg->stat->events[i], cpu);
 
-               per_cpu(mem->stat->events[i], cpu) = 0;
-               mem->nocpu_base.events[i] += x;
+               per_cpu(memcg->stat->events[i], cpu) = 0;
+               memcg->nocpu_base.events[i] += x;
        }
        /* need to clear ON_MOVE value, works as a kind of lock. */
-       per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
-       spin_unlock(&mem->pcp_counter_lock);
+       per_cpu(memcg->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
+       spin_unlock(&memcg->pcp_counter_lock);
 }
 
-static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
+static void synchronize_mem_cgroup_on_move(struct mem_cgroup *memcg, int cpu)
 {
        int idx = MEM_CGROUP_ON_MOVE;
 
-       spin_lock(&mem->pcp_counter_lock);
-       per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
-       spin_unlock(&mem->pcp_counter_lock);
+       spin_lock(&memcg->pcp_counter_lock);
+       per_cpu(memcg->stat->count[idx], cpu) = memcg->nocpu_base.count[idx];
+       spin_unlock(&memcg->pcp_counter_lock);
 }
 
 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
@@ -2084,7 +2090,7 @@ static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
        struct mem_cgroup *iter;
 
        if ((action == CPU_ONLINE)) {
-               for_each_mem_cgroup_all(iter)
+               for_each_mem_cgroup(iter)
                        synchronize_mem_cgroup_on_move(iter, cpu);
                return NOTIFY_OK;
        }
@@ -2092,7 +2098,7 @@ static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
        if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
                return NOTIFY_OK;
 
-       for_each_mem_cgroup_all(iter)
+       for_each_mem_cgroup(iter)
                mem_cgroup_drain_pcp_counter(iter, cpu);
 
        stock = &per_cpu(memcg_stock, cpu);
@@ -2110,7 +2116,7 @@ enum {
        CHARGE_OOM_DIE,         /* the current is killed because of OOM */
 };
 
-static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
+static int mem_cgroup_do_charge(struct mem_cgroup *memcg, gfp_t gfp_mask,
                                unsigned int nr_pages, bool oom_check)
 {
        unsigned long csize = nr_pages * PAGE_SIZE;
@@ -2119,16 +2125,16 @@ static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
        unsigned long flags = 0;
        int ret;
 
-       ret = res_counter_charge(&mem->res, csize, &fail_res);
+       ret = res_counter_charge(&memcg->res, csize, &fail_res);
 
        if (likely(!ret)) {
                if (!do_swap_account)
                        return CHARGE_OK;
-               ret = res_counter_charge(&mem->memsw, csize, &fail_res);
+               ret = res_counter_charge(&memcg->memsw, csize, &fail_res);
                if (likely(!ret))
                        return CHARGE_OK;
 
-               res_counter_uncharge(&mem->res, csize);
+               res_counter_uncharge(&memcg->res, csize);
                mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
                flags |= MEM_CGROUP_RECLAIM_NOSWAP;
        } else
@@ -2146,8 +2152,7 @@ static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
        if (!(gfp_mask & __GFP_WAIT))
                return CHARGE_WOULDBLOCK;
 
-       ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
-                                             gfp_mask, flags, NULL);
+       ret = mem_cgroup_reclaim(mem_over_limit, gfp_mask, flags);
        if (mem_cgroup_margin(mem_over_limit) >= nr_pages)
                return CHARGE_RETRY;
        /*
@@ -2180,18 +2185,35 @@ static int mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
 }
 
 /*
- * Unlike exported interface, "oom" parameter is added. if oom==true,
- * oom-killer can be invoked.
+ * __mem_cgroup_try_charge() does
+ * 1. detect memcg to be charged against from passed *mm and *ptr,
+ * 2. update res_counter
+ * 3. call memory reclaim if necessary.
+ *
+ * In some special case, if the task is fatal, fatal_signal_pending() or
+ * has TIF_MEMDIE, this function returns -EINTR while writing root_mem_cgroup
+ * to *ptr. There are two reasons for this. 1: fatal threads should quit as soon
+ * as possible without any hazards. 2: all pages should have a valid
+ * pc->mem_cgroup. If mm is NULL and the caller doesn't pass a valid memcg
+ * pointer, that is treated as a charge to root_mem_cgroup.
+ *
+ * So __mem_cgroup_try_charge() will return
+ *  0       ...  on success, filling *ptr with a valid memcg pointer.
+ *  -ENOMEM ...  charge failure because of resource limits.
+ *  -EINTR  ...  if thread is fatal. *ptr is filled with root_mem_cgroup.
+ *
+ * Unlike the exported interface, an "oom" parameter is added. if oom==true,
+ * the oom-killer can be invoked.
  */
 static int __mem_cgroup_try_charge(struct mm_struct *mm,
                                   gfp_t gfp_mask,
                                   unsigned int nr_pages,
-                                  struct mem_cgroup **memcg,
+                                  struct mem_cgroup **ptr,
                                   bool oom)
 {
        unsigned int batch = max(CHARGE_BATCH, nr_pages);
        int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        int ret;
 
        /*
@@ -2209,17 +2231,17 @@ static int __mem_cgroup_try_charge(struct mm_struct *mm,
         * thread group leader migrates. It's possible that mm is not
         * set, if so charge the init_mm (happens for pagecache usage).
         */
-       if (!*memcg && !mm)
-               goto bypass;
+       if (!*ptr && !mm)
+               *ptr = root_mem_cgroup;
 again:
-       if (*memcg) { /* css should be a valid one */
-               mem = *memcg;
-               VM_BUG_ON(css_is_removed(&mem->css));
-               if (mem_cgroup_is_root(mem))
+       if (*ptr) { /* css should be a valid one */
+               memcg = *ptr;
+               VM_BUG_ON(css_is_removed(&memcg->css));
+               if (mem_cgroup_is_root(memcg))
                        goto done;
-               if (nr_pages == 1 && consume_stock(mem))
+               if (nr_pages == 1 && consume_stock(memcg))
                        goto done;
-               css_get(&mem->css);
+               css_get(&memcg->css);
        } else {
                struct task_struct *p;
 
@@ -2227,7 +2249,7 @@ again:
                p = rcu_dereference(mm->owner);
                /*
                 * Because we don't have task_lock(), "p" can exit.
-                * In that case, "mem" can point to root or p can be NULL with
+                * In that case, "memcg" can point to root or p can be NULL with
                 * race with swapoff. Then, we have small risk of mis-accouning.
                 * But such kind of mis-account by race always happens because
                 * we don't have cgroup_mutex(). It's overkill and we allo that
@@ -2235,12 +2257,14 @@ again:
                 * (*) swapoff at el will charge against mm-struct not against
                 * task-struct. So, mm->owner can be NULL.
                 */
-               mem = mem_cgroup_from_task(p);
-               if (!mem || mem_cgroup_is_root(mem)) {
+               memcg = mem_cgroup_from_task(p);
+               if (!memcg)
+                       memcg = root_mem_cgroup;
+               if (mem_cgroup_is_root(memcg)) {
                        rcu_read_unlock();
                        goto done;
                }
-               if (nr_pages == 1 && consume_stock(mem)) {
+               if (nr_pages == 1 && consume_stock(memcg)) {
                        /*
                         * It seems dagerous to access memcg without css_get().
                         * But considering how consume_stok works, it's not
@@ -2253,7 +2277,7 @@ again:
                        goto done;
                }
                /* after here, we may be blocked. we need to get refcnt */
-               if (!css_tryget(&mem->css)) {
+               if (!css_tryget(&memcg->css)) {
                        rcu_read_unlock();
                        goto again;
                }
@@ -2265,7 +2289,7 @@ again:
 
                /* If killed, bypass charge */
                if (fatal_signal_pending(current)) {
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto bypass;
                }
 
@@ -2275,44 +2299,44 @@ again:
                        nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
                }
 
-               ret = mem_cgroup_do_charge(mem, gfp_mask, batch, oom_check);
+               ret = mem_cgroup_do_charge(memcg, gfp_mask, batch, oom_check);
                switch (ret) {
                case CHARGE_OK:
                        break;
                case CHARGE_RETRY: /* not in OOM situation but retry */
                        batch = nr_pages;
-                       css_put(&mem->css);
-                       mem = NULL;
+                       css_put(&memcg->css);
+                       memcg = NULL;
                        goto again;
                case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto nomem;
                case CHARGE_NOMEM: /* OOM routine works */
                        if (!oom) {
-                               css_put(&mem->css);
+                               css_put(&memcg->css);
                                goto nomem;
                        }
                        /* If oom, we never return -ENOMEM */
                        nr_oom_retries--;
                        break;
                case CHARGE_OOM_DIE: /* Killed by OOM Killer */
-                       css_put(&mem->css);
+                       css_put(&memcg->css);
                        goto bypass;
                }
        } while (ret != CHARGE_OK);
 
        if (batch > nr_pages)
-               refill_stock(mem, batch - nr_pages);
-       css_put(&mem->css);
+               refill_stock(memcg, batch - nr_pages);
+       css_put(&memcg->css);
 done:
-       *memcg = mem;
+       *ptr = memcg;
        return 0;
 nomem:
-       *memcg = NULL;
+       *ptr = NULL;
        return -ENOMEM;
 bypass:
-       *memcg = NULL;
-       return 0;
+       *ptr = root_mem_cgroup;
+       return -EINTR;
 }
 
 /*
@@ -2320,15 +2344,15 @@ bypass:
  * This function is for that and do uncharge, put css's refcnt.
  * gotten by try_charge().
  */
-static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
+static void __mem_cgroup_cancel_charge(struct mem_cgroup *memcg,
                                       unsigned int nr_pages)
 {
-       if (!mem_cgroup_is_root(mem)) {
+       if (!mem_cgroup_is_root(memcg)) {
                unsigned long bytes = nr_pages * PAGE_SIZE;
 
-               res_counter_uncharge(&mem->res, bytes);
+               res_counter_uncharge(&memcg->res, bytes);
                if (do_swap_account)
-                       res_counter_uncharge(&mem->memsw, bytes);
+                       res_counter_uncharge(&memcg->memsw, bytes);
        }
 }
 
@@ -2353,7 +2377,7 @@ static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
 
 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        struct page_cgroup *pc;
        unsigned short id;
        swp_entry_t ent;
@@ -2363,23 +2387,23 @@ struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
        pc = lookup_page_cgroup(page);
        lock_page_cgroup(pc);
        if (PageCgroupUsed(pc)) {
-               mem = pc->mem_cgroup;
-               if (mem && !css_tryget(&mem->css))
-                       mem = NULL;
+               memcg = pc->mem_cgroup;
+               if (memcg && !css_tryget(&memcg->css))
+                       memcg = NULL;
        } else if (PageSwapCache(page)) {
                ent.val = page_private(page);
-               id = lookup_swap_cgroup(ent);
+               id = lookup_swap_cgroup_id(ent);
                rcu_read_lock();
-               mem = mem_cgroup_lookup(id);
-               if (mem && !css_tryget(&mem->css))
-                       mem = NULL;
+               memcg = mem_cgroup_lookup(id);
+               if (memcg && !css_tryget(&memcg->css))
+                       memcg = NULL;
                rcu_read_unlock();
        }
        unlock_page_cgroup(pc);
-       return mem;
+       return memcg;
 }
 
-static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
+static void __mem_cgroup_commit_charge(struct mem_cgroup *memcg,
                                       struct page *page,
                                       unsigned int nr_pages,
                                       struct page_cgroup *pc,
@@ -2388,14 +2412,14 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
        lock_page_cgroup(pc);
        if (unlikely(PageCgroupUsed(pc))) {
                unlock_page_cgroup(pc);
-               __mem_cgroup_cancel_charge(mem, nr_pages);
+               __mem_cgroup_cancel_charge(memcg, nr_pages);
                return;
        }
        /*
         * we don't need page_cgroup_lock about tail pages, becase they are not
         * accessed by any other context at this point.
         */
-       pc->mem_cgroup = mem;
+       pc->mem_cgroup = memcg;
        /*
         * We access a page_cgroup asynchronously without lock_page_cgroup().
         * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
@@ -2418,57 +2442,43 @@ static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
                break;
        }
 
-       mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), nr_pages);
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), nr_pages);
        unlock_page_cgroup(pc);
+       WARN_ON_ONCE(PageLRU(page));
        /*
         * "charge_statistics" updated event counter. Then, check it.
         * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
         * if they exceeds softlimit.
         */
-       memcg_check_events(mem, page);
+       memcg_check_events(memcg, page);
 }
 
 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 
 #define PCGF_NOCOPY_AT_SPLIT ((1 << PCG_LOCK) | (1 << PCG_MOVE_LOCK) |\
-                       (1 << PCG_ACCT_LRU) | (1 << PCG_MIGRATION))
+                       (1 << PCG_MIGRATION))
 /*
  * Because tail pages are not marked as "used", set it. We're under
- * zone->lru_lock, 'splitting on pmd' and compund_lock.
+ * zone->lru_lock, 'splitting on pmd' and compound_lock.
+ * charge/uncharge will be never happen and move_account() is done under
+ * compound_lock(), so we don't have to take care of races.
  */
-void mem_cgroup_split_huge_fixup(struct page *head, struct page *tail)
+void mem_cgroup_split_huge_fixup(struct page *head)
 {
        struct page_cgroup *head_pc = lookup_page_cgroup(head);
-       struct page_cgroup *tail_pc = lookup_page_cgroup(tail);
-       unsigned long flags;
+       struct page_cgroup *pc;
+       int i;
 
        if (mem_cgroup_disabled())
                return;
-       /*
-        * We have no races with charge/uncharge but will have races with
-        * page state accounting.
-        */
-       move_lock_page_cgroup(head_pc, &flags);
-
-       tail_pc->mem_cgroup = head_pc->mem_cgroup;
-       smp_wmb(); /* see __commit_charge() */
-       if (PageCgroupAcctLRU(head_pc)) {
-               enum lru_list lru;
-               struct mem_cgroup_per_zone *mz;
-
-               /*
-                * LRU flags cannot be copied because we need to add tail
-                *.page to LRU by generic call and our hook will be called.
-                * We hold lru_lock, then, reduce counter directly.
-                */
-               lru = page_lru(head);
-               mz = page_cgroup_zoneinfo(head_pc->mem_cgroup, head);
-               MEM_CGROUP_ZSTAT(mz, lru) -= 1;
+       for (i = 1; i < HPAGE_PMD_NR; i++) {
+               pc = head_pc + i;
+               pc->mem_cgroup = head_pc->mem_cgroup;
+               smp_wmb();/* see __commit_charge() */
+               pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
        }
-       tail_pc->flags = head_pc->flags & ~PCGF_NOCOPY_AT_SPLIT;
-       move_unlock_page_cgroup(head_pc, &flags);
 }
-#endif
+#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
 
 /**
  * mem_cgroup_move_account - move account of the page
@@ -2583,7 +2593,7 @@ static int mem_cgroup_move_parent(struct page *page,
 
        parent = mem_cgroup_from_cont(pcg);
        ret = __mem_cgroup_try_charge(NULL, gfp_mask, nr_pages, &parent, false);
-       if (ret || !parent)
+       if (ret)
                goto put_back;
 
        if (nr_pages > 1)
@@ -2612,7 +2622,7 @@ out:
 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
                                gfp_t gfp_mask, enum charge_type ctype)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        unsigned int nr_pages = 1;
        struct page_cgroup *pc;
        bool oom = true;
@@ -2629,13 +2639,10 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
        }
 
        pc = lookup_page_cgroup(page);
-       BUG_ON(!pc); /* XXX: remove this and move pc lookup into commit */
-
-       ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &mem, oom);
-       if (ret || !mem)
+       ret = __mem_cgroup_try_charge(mm, gfp_mask, nr_pages, &memcg, oom);
+       if (ret == -ENOMEM)
                return ret;
-
-       __mem_cgroup_commit_charge(mem, page, nr_pages, pc, ctype);
+       __mem_cgroup_commit_charge(memcg, page, nr_pages, pc, ctype);
        return 0;
 }
 
@@ -2644,19 +2651,11 @@ int mem_cgroup_newpage_charge(struct page *page,
 {
        if (mem_cgroup_disabled())
                return 0;
-       /*
-        * If already mapped, we don't have to account.
-        * If page cache, page->mapping has address_space.
-        * But page->mapping may have out-of-use anon_vma pointer,
-        * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
-        * is NULL.
-        */
-       if (page_mapped(page) || (page->mapping && !PageAnon(page)))
-               return 0;
-       if (unlikely(!mm))
-               mm = &init_mm;
+       VM_BUG_ON(page_mapped(page));
+       VM_BUG_ON(page->mapping && !PageAnon(page));
+       VM_BUG_ON(!mm);
        return mem_cgroup_charge_common(page, mm, gfp_mask,
-                               MEM_CGROUP_CHARGE_TYPE_MAPPED);
+                                       MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
 static void
@@ -2664,83 +2663,58 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
                                        enum charge_type ctype);
 
 static void
-__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *mem,
+__mem_cgroup_commit_charge_lrucare(struct page *page, struct mem_cgroup *memcg,
                                        enum charge_type ctype)
 {
        struct page_cgroup *pc = lookup_page_cgroup(page);
+       struct zone *zone = page_zone(page);
+       unsigned long flags;
+       bool removed = false;
+
        /*
         * In some case, SwapCache, FUSE(splice_buf->radixtree), the page
         * is already on LRU. It means the page may on some other page_cgroup's
         * LRU. Take care of it.
         */
-       mem_cgroup_lru_del_before_commit(page);
-       __mem_cgroup_commit_charge(mem, page, 1, pc, ctype);
-       mem_cgroup_lru_add_after_commit(page);
+       spin_lock_irqsave(&zone->lru_lock, flags);
+       if (PageLRU(page)) {
+               del_page_from_lru_list(zone, page, page_lru(page));
+               ClearPageLRU(page);
+               removed = true;
+       }
+       __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
+       if (removed) {
+               add_page_to_lru_list(zone, page, page_lru(page));
+               SetPageLRU(page);
+       }
+       spin_unlock_irqrestore(&zone->lru_lock, flags);
        return;
 }
 
 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
                                gfp_t gfp_mask)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
+       enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
        int ret;
 
        if (mem_cgroup_disabled())
                return 0;
        if (PageCompound(page))
                return 0;
-       /*
-        * Corner case handling. This is called from add_to_page_cache()
-        * in usual. But some FS (shmem) precharges this page before calling it
-        * and call add_to_page_cache() with GFP_NOWAIT.
-        *
-        * For GFP_NOWAIT case, the page may be pre-charged before calling
-        * add_to_page_cache(). (See shmem.c) check it here and avoid to call
-        * charge twice. (It works but has to pay a bit larger cost.)
-        * And when the page is SwapCache, it should take swap information
-        * into account. This is under lock_page() now.
-        */
-       if (!(gfp_mask & __GFP_WAIT)) {
-               struct page_cgroup *pc;
-
-               pc = lookup_page_cgroup(page);
-               if (!pc)
-                       return 0;
-               lock_page_cgroup(pc);
-               if (PageCgroupUsed(pc)) {
-                       unlock_page_cgroup(pc);
-                       return 0;
-               }
-               unlock_page_cgroup(pc);
-       }
 
        if (unlikely(!mm))
                mm = &init_mm;
+       if (!page_is_file_cache(page))
+               type = MEM_CGROUP_CHARGE_TYPE_SHMEM;
 
-       if (page_is_file_cache(page)) {
-               ret = __mem_cgroup_try_charge(mm, gfp_mask, 1, &mem, true);
-               if (ret || !mem)
-                       return ret;
-
-               /*
-                * FUSE reuses pages without going through the final
-                * put that would remove them from the LRU list, make
-                * sure that they get relinked properly.
-                */
-               __mem_cgroup_commit_charge_lrucare(page, mem,
-                                       MEM_CGROUP_CHARGE_TYPE_CACHE);
-               return ret;
-       }
-       /* shmem */
-       if (PageSwapCache(page)) {
-               ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
+       if (!PageSwapCache(page))
+               ret = mem_cgroup_charge_common(page, mm, gfp_mask, type);
+       else { /* page is swapcache/shmem */
+               ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &memcg);
                if (!ret)
-                       __mem_cgroup_commit_charge_swapin(page, mem,
-                                       MEM_CGROUP_CHARGE_TYPE_SHMEM);
-       } else
-               ret = mem_cgroup_charge_common(page, mm, gfp_mask,
-                                       MEM_CGROUP_CHARGE_TYPE_SHMEM);
-
+                       __mem_cgroup_commit_charge_swapin(page, memcg, type);
+       }
        return ret;
 }
 
@@ -2752,12 +2726,12 @@ int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
  */
 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
                                 struct page *page,
-                                gfp_t mask, struct mem_cgroup **ptr)
+                                gfp_t mask, struct mem_cgroup **memcgp)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        int ret;
 
-       *ptr = NULL;
+       *memcgp = NULL;
 
        if (mem_cgroup_disabled())
                return 0;
@@ -2772,30 +2746,35 @@ int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
         */
        if (!PageSwapCache(page))
                goto charge_cur_mm;
-       mem = try_get_mem_cgroup_from_page(page);
-       if (!mem)
+       memcg = try_get_mem_cgroup_from_page(page);
+       if (!memcg)
                goto charge_cur_mm;
-       *ptr = mem;
-       ret = __mem_cgroup_try_charge(NULL, mask, 1, ptr, true);
-       css_put(&mem->css);
+       *memcgp = memcg;
+       ret = __mem_cgroup_try_charge(NULL, mask, 1, memcgp, true);
+       css_put(&memcg->css);
+       if (ret == -EINTR)
+               ret = 0;
        return ret;
 charge_cur_mm:
        if (unlikely(!mm))
                mm = &init_mm;
-       return __mem_cgroup_try_charge(mm, mask, 1, ptr, true);
+       ret = __mem_cgroup_try_charge(mm, mask, 1, memcgp, true);
+       if (ret == -EINTR)
+               ret = 0;
+       return ret;
 }
 
 static void
-__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
+__mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *memcg,
                                        enum charge_type ctype)
 {
        if (mem_cgroup_disabled())
                return;
-       if (!ptr)
+       if (!memcg)
                return;
-       cgroup_exclude_rmdir(&ptr->css);
+       cgroup_exclude_rmdir(&memcg->css);
 
-       __mem_cgroup_commit_charge_lrucare(page, ptr, ctype);
+       __mem_cgroup_commit_charge_lrucare(page, memcg, ctype);
        /*
         * Now swap is on-memory. This means this page may be
         * counted both as mem and swap....double count.
@@ -2805,21 +2784,22 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
         */
        if (do_swap_account && PageSwapCache(page)) {
                swp_entry_t ent = {.val = page_private(page)};
+               struct mem_cgroup *swap_memcg;
                unsigned short id;
-               struct mem_cgroup *memcg;
 
                id = swap_cgroup_record(ent, 0);
                rcu_read_lock();
-               memcg = mem_cgroup_lookup(id);
-               if (memcg) {
+               swap_memcg = mem_cgroup_lookup(id);
+               if (swap_memcg) {
                        /*
                         * This recorded memcg can be obsolete one. So, avoid
                         * calling css_tryget
                         */
-                       if (!mem_cgroup_is_root(memcg))
-                               res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
-                       mem_cgroup_swap_statistics(memcg, false);
-                       mem_cgroup_put(memcg);
+                       if (!mem_cgroup_is_root(swap_memcg))
+                               res_counter_uncharge(&swap_memcg->memsw,
+                                                    PAGE_SIZE);
+                       mem_cgroup_swap_statistics(swap_memcg, false);
+                       mem_cgroup_put(swap_memcg);
                }
                rcu_read_unlock();
        }
@@ -2828,25 +2808,26 @@ __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
         * So, rmdir()->pre_destroy() can be called while we do this charge.
         * In that case, we need to call pre_destroy() again. check it here.
         */
-       cgroup_release_and_wakeup_rmdir(&ptr->css);
+       cgroup_release_and_wakeup_rmdir(&memcg->css);
 }
 
-void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
+void mem_cgroup_commit_charge_swapin(struct page *page,
+                                    struct mem_cgroup *memcg)
 {
-       __mem_cgroup_commit_charge_swapin(page, ptr,
-                                       MEM_CGROUP_CHARGE_TYPE_MAPPED);
+       __mem_cgroup_commit_charge_swapin(page, memcg,
+                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
-void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
+void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *memcg)
 {
        if (mem_cgroup_disabled())
                return;
-       if (!mem)
+       if (!memcg)
                return;
-       __mem_cgroup_cancel_charge(mem, 1);
+       __mem_cgroup_cancel_charge(memcg, 1);
 }
 
-static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
+static void mem_cgroup_do_uncharge(struct mem_cgroup *memcg,
                                   unsigned int nr_pages,
                                   const enum charge_type ctype)
 {
@@ -2864,7 +2845,7 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
         * uncharges. Then, it's ok to ignore memcg's refcnt.
         */
        if (!batch->memcg)
-               batch->memcg = mem;
+               batch->memcg = memcg;
        /*
         * do_batch > 0 when unmapping pages or inode invalidate/truncate.
         * In those cases, all pages freed continuously can be expected to be in
@@ -2884,7 +2865,7 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
         * merge a series of uncharges to an uncharge of res_counter.
         * If not, we uncharge res_counter ony by one.
         */
-       if (batch->memcg != mem)
+       if (batch->memcg != memcg)
                goto direct_uncharge;
        /* remember freed charge and uncharge it later */
        batch->nr_pages++;
@@ -2892,11 +2873,11 @@ static void mem_cgroup_do_uncharge(struct mem_cgroup *mem,
                batch->memsw_nr_pages++;
        return;
 direct_uncharge:
-       res_counter_uncharge(&mem->res, nr_pages * PAGE_SIZE);
+       res_counter_uncharge(&memcg->res, nr_pages * PAGE_SIZE);
        if (uncharge_memsw)
-               res_counter_uncharge(&mem->memsw, nr_pages * PAGE_SIZE);
-       if (unlikely(batch->memcg != mem))
-               memcg_oom_recover(mem);
+               res_counter_uncharge(&memcg->memsw, nr_pages * PAGE_SIZE);
+       if (unlikely(batch->memcg != memcg))
+               memcg_oom_recover(memcg);
        return;
 }
 
@@ -2906,7 +2887,7 @@ direct_uncharge:
 static struct mem_cgroup *
 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        unsigned int nr_pages = 1;
        struct page_cgroup *pc;
 
@@ -2924,12 +2905,12 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
         * Check if our page_cgroup is valid
         */
        pc = lookup_page_cgroup(page);
-       if (unlikely(!pc || !PageCgroupUsed(pc)))
+       if (unlikely(!PageCgroupUsed(pc)))
                return NULL;
 
        lock_page_cgroup(pc);
 
-       mem = pc->mem_cgroup;
+       memcg = pc->mem_cgroup;
 
        if (!PageCgroupUsed(pc))
                goto unlock_out;
@@ -2952,7 +2933,7 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
                break;
        }
 
-       mem_cgroup_charge_statistics(mem, PageCgroupCache(pc), -nr_pages);
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -nr_pages);
 
        ClearPageCgroupUsed(pc);
        /*
@@ -2964,18 +2945,18 @@ __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
 
        unlock_page_cgroup(pc);
        /*
-        * even after unlock, we have mem->res.usage here and this memcg
+        * even after unlock, we have memcg->res.usage here and this memcg
         * will never be freed.
         */
-       memcg_check_events(mem, page);
+       memcg_check_events(memcg, page);
        if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
-               mem_cgroup_swap_statistics(mem, true);
-               mem_cgroup_get(mem);
+               mem_cgroup_swap_statistics(memcg, true);
+               mem_cgroup_get(memcg);
        }
-       if (!mem_cgroup_is_root(mem))
-               mem_cgroup_do_uncharge(mem, nr_pages, ctype);
+       if (!mem_cgroup_is_root(memcg))
+               mem_cgroup_do_uncharge(memcg, nr_pages, ctype);
 
-       return mem;
+       return memcg;
 
 unlock_out:
        unlock_page_cgroup(pc);
@@ -2987,8 +2968,7 @@ void mem_cgroup_uncharge_page(struct page *page)
        /* early check. */
        if (page_mapped(page))
                return;
-       if (page->mapping && !PageAnon(page))
-               return;
+       VM_BUG_ON(page->mapping && !PageAnon(page));
        __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
 }
 
@@ -3046,6 +3026,23 @@ void mem_cgroup_uncharge_end(void)
        batch->memcg = NULL;
 }
 
+/*
+ * A function for resetting pc->mem_cgroup for newly allocated pages.
+ * This function should be called if the newpage will be added to LRU
+ * before start accounting.
+ */
+void mem_cgroup_reset_owner(struct page *newpage)
+{
+       struct page_cgroup *pc;
+
+       if (mem_cgroup_disabled())
+               return;
+
+       pc = lookup_page_cgroup(newpage);
+       VM_BUG_ON(PageCgroupUsed(pc));
+       pc->mem_cgroup = root_mem_cgroup;
+}
+
 #ifdef CONFIG_SWAP
 /*
  * called after __delete_from_swap_cache() and drop "page" account.
@@ -3163,14 +3160,14 @@ static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
  * page belongs to.
  */
 int mem_cgroup_prepare_migration(struct page *page,
-       struct page *newpage, struct mem_cgroup **ptr, gfp_t gfp_mask)
+       struct page *newpage, struct mem_cgroup **memcgp, gfp_t gfp_mask)
 {
-       struct mem_cgroup *mem = NULL;
+       struct mem_cgroup *memcg = NULL;
        struct page_cgroup *pc;
        enum charge_type ctype;
        int ret = 0;
 
-       *ptr = NULL;
+       *memcgp = NULL;
 
        VM_BUG_ON(PageTransHuge(page));
        if (mem_cgroup_disabled())
@@ -3179,8 +3176,8 @@ int mem_cgroup_prepare_migration(struct page *page,
        pc = lookup_page_cgroup(page);
        lock_page_cgroup(pc);
        if (PageCgroupUsed(pc)) {
-               mem = pc->mem_cgroup;
-               css_get(&mem->css);
+               memcg = pc->mem_cgroup;
+               css_get(&memcg->css);
                /*
                 * At migrating an anonymous page, its mapcount goes down
                 * to 0 and uncharge() will be called. But, even if it's fully
@@ -3218,13 +3215,13 @@ int mem_cgroup_prepare_migration(struct page *page,
         * If the page is not charged at this point,
         * we return here.
         */
-       if (!mem)
+       if (!memcg)
                return 0;
 
-       *ptr = mem;
-       ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, ptr, false);
-       css_put(&mem->css);/* drop extra refcnt */
-       if (ret || *ptr == NULL) {
+       *memcgp = memcg;
+       ret = __mem_cgroup_try_charge(NULL, gfp_mask, 1, memcgp, false);
+       css_put(&memcg->css);/* drop extra refcnt */
+       if (ret) {
                if (PageAnon(page)) {
                        lock_page_cgroup(pc);
                        ClearPageCgroupMigration(pc);
@@ -3234,6 +3231,7 @@ int mem_cgroup_prepare_migration(struct page *page,
                         */
                        mem_cgroup_uncharge_page(page);
                }
+               /* we'll need to revisit this error code (we have -EINTR) */
                return -ENOMEM;
        }
        /*
@@ -3249,21 +3247,21 @@ int mem_cgroup_prepare_migration(struct page *page,
                ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
        else
                ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
-       __mem_cgroup_commit_charge(mem, page, 1, pc, ctype);
+       __mem_cgroup_commit_charge(memcg, page, 1, pc, ctype);
        return ret;
 }
 
 /* remove redundant charge if migration failed*/
-void mem_cgroup_end_migration(struct mem_cgroup *mem,
+void mem_cgroup_end_migration(struct mem_cgroup *memcg,
        struct page *oldpage, struct page *newpage, bool migration_ok)
 {
        struct page *used, *unused;
        struct page_cgroup *pc;
 
-       if (!mem)
+       if (!memcg)
                return;
        /* blocks rmdir() */
-       cgroup_exclude_rmdir(&mem->css);
+       cgroup_exclude_rmdir(&memcg->css);
        if (!migration_ok) {
                used = oldpage;
                unused = newpage;
@@ -3299,32 +3297,41 @@ void mem_cgroup_end_migration(struct mem_cgroup *mem,
         * So, rmdir()->pre_destroy() can be called while we do this charge.
         * In that case, we need to call pre_destroy() again. check it here.
         */
-       cgroup_release_and_wakeup_rmdir(&mem->css);
+       cgroup_release_and_wakeup_rmdir(&memcg->css);
 }
 
 /*
- * A call to try to shrink memory usage on charge failure at shmem's swapin.
- * Calling hierarchical_reclaim is not enough because we should update
- * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
- * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
- * not from the memcg which this page would be charged to.
- * try_charge_swapin does all of these works properly.
+ * At replace page cache, newpage is not under any memcg but it's on
+ * LRU. So, this function doesn't touch res_counter but handles LRU
+ * in correct way. Both pages are locked so we cannot race with uncharge.
  */
-int mem_cgroup_shmem_charge_fallback(struct page *page,
-                           struct mm_struct *mm,
-                           gfp_t gfp_mask)
+void mem_cgroup_replace_page_cache(struct page *oldpage,
+                                 struct page *newpage)
 {
-       struct mem_cgroup *mem;
-       int ret;
+       struct mem_cgroup *memcg;
+       struct page_cgroup *pc;
+       enum charge_type type = MEM_CGROUP_CHARGE_TYPE_CACHE;
 
        if (mem_cgroup_disabled())
-               return 0;
+               return;
 
-       ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
-       if (!ret)
-               mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
+       pc = lookup_page_cgroup(oldpage);
+       /* fix accounting on old pages */
+       lock_page_cgroup(pc);
+       memcg = pc->mem_cgroup;
+       mem_cgroup_charge_statistics(memcg, PageCgroupCache(pc), -1);
+       ClearPageCgroupUsed(pc);
+       unlock_page_cgroup(pc);
 
-       return ret;
+       if (PageSwapBacked(oldpage))
+               type = MEM_CGROUP_CHARGE_TYPE_SHMEM;
+
+       /*
+        * Even if newpage->mapping was NULL before starting replacement,
+        * the newpage may be on LRU(or pagevec for LRU) already. We lock
+        * LRU while we overwrite pc->mem_cgroup.
+        */
+       __mem_cgroup_commit_charge_lrucare(newpage, memcg, type);
 }
 
 #ifdef CONFIG_DEBUG_VM
@@ -3333,6 +3340,11 @@ static struct page_cgroup *lookup_page_cgroup_used(struct page *page)
        struct page_cgroup *pc;
 
        pc = lookup_page_cgroup(page);
+       /*
+        * Can be NULL while feeding pages into the page allocator for
+        * the first time, i.e. during boot or memory hotplug;
+        * or when mem_cgroup_disabled().
+        */
        if (likely(pc) && PageCgroupUsed(pc))
                return pc;
        return NULL;
@@ -3352,23 +3364,8 @@ void mem_cgroup_print_bad_page(struct page *page)
 
        pc = lookup_page_cgroup_used(page);
        if (pc) {
-               int ret = -1;
-               char *path;
-
-               printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p",
+               printk(KERN_ALERT "pc:%p pc->flags:%lx pc->mem_cgroup:%p\n",
                       pc, pc->flags, pc->mem_cgroup);
-
-               path = kmalloc(PATH_MAX, GFP_KERNEL);
-               if (path) {
-                       rcu_read_lock();
-                       ret = cgroup_path(pc->mem_cgroup->css.cgroup,
-                                                       path, PATH_MAX);
-                       rcu_read_unlock();
-               }
-
-               printk(KERN_CONT "(%s)\n",
-                               (ret < 0) ? "cannot get the path" : path);
-               kfree(path);
        }
 }
 #endif
@@ -3403,7 +3400,7 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                /*
                 * Rather than hide all in some function, I do this in
                 * open coded manner. You see what this really does.
-                * We have to guarantee mem->res.limit < mem->memsw.limit.
+                * We have to guarantee memcg->res.limit < memcg->memsw.limit.
                 */
                mutex_lock(&set_limit_mutex);
                memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
@@ -3429,9 +3426,8 @@ static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
                if (!ret)
                        break;
 
-               mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
-                                               MEM_CGROUP_RECLAIM_SHRINK,
-                                               NULL);
+               mem_cgroup_reclaim(memcg, GFP_KERNEL,
+                                  MEM_CGROUP_RECLAIM_SHRINK);
                curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
                /* Usage is reduced ? */
                if (curusage >= oldusage)
@@ -3465,7 +3461,7 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
                /*
                 * Rather than hide all in some function, I do this in
                 * open coded manner. You see what this really does.
-                * We have to guarantee mem->res.limit < mem->memsw.limit.
+                * We have to guarantee memcg->res.limit < memcg->memsw.limit.
                 */
                mutex_lock(&set_limit_mutex);
                memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
@@ -3489,10 +3485,9 @@ static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
                if (!ret)
                        break;
 
-               mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
-                                               MEM_CGROUP_RECLAIM_NOSWAP |
-                                               MEM_CGROUP_RECLAIM_SHRINK,
-                                               NULL);
+               mem_cgroup_reclaim(memcg, GFP_KERNEL,
+                                  MEM_CGROUP_RECLAIM_NOSWAP |
+                                  MEM_CGROUP_RECLAIM_SHRINK);
                curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
                /* Usage is reduced ? */
                if (curusage >= oldusage)
@@ -3535,10 +3530,8 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
                        break;
 
                nr_scanned = 0;
-               reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
-                                               gfp_mask,
-                                               MEM_CGROUP_RECLAIM_SOFT,
-                                               &nr_scanned);
+               reclaimed = mem_cgroup_soft_reclaim(mz->mem, zone,
+                                                   gfp_mask, &nr_scanned);
                nr_reclaimed += reclaimed;
                *total_scanned += nr_scanned;
                spin_lock(&mctz->lock);
@@ -3603,25 +3596,26 @@ unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
  * This routine traverse page_cgroup in given list and drop them all.
  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
  */
-static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
+static int mem_cgroup_force_empty_list(struct mem_cgroup *memcg,
                                int node, int zid, enum lru_list lru)
 {
-       struct zone *zone;
        struct mem_cgroup_per_zone *mz;
-       struct page_cgroup *pc, *busy;
        unsigned long flags, loop;
        struct list_head *list;
+       struct page *busy;
+       struct zone *zone;
        int ret = 0;
 
        zone = &NODE_DATA(node)->node_zones[zid];
-       mz = mem_cgroup_zoneinfo(mem, node, zid);
-       list = &mz->lists[lru];
+       mz = mem_cgroup_zoneinfo(memcg, node, zid);
+       list = &mz->lruvec.lists[lru];
 
        loop = MEM_CGROUP_ZSTAT(mz, lru);
        /* give some margin against EBUSY etc...*/
        loop += 256;
        busy = NULL;
        while (loop--) {
+               struct page_cgroup *pc;
                struct page *page;
 
                ret = 0;
@@ -3630,24 +3624,24 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
                        spin_unlock_irqrestore(&zone->lru_lock, flags);
                        break;
                }
-               pc = list_entry(list->prev, struct page_cgroup, lru);
-               if (busy == pc) {
-                       list_move(&pc->lru, list);
+               page = list_entry(list->prev, struct page, lru);
+               if (busy == page) {
+                       list_move(&page->lru, list);
                        busy = NULL;
                        spin_unlock_irqrestore(&zone->lru_lock, flags);
                        continue;
                }
                spin_unlock_irqrestore(&zone->lru_lock, flags);
 
-               page = lookup_cgroup_page(pc);
+               pc = lookup_page_cgroup(page);
 
-               ret = mem_cgroup_move_parent(page, pc, mem, GFP_KERNEL);
-               if (ret == -ENOMEM)
+               ret = mem_cgroup_move_parent(page, pc, memcg, GFP_KERNEL);
+               if (ret == -ENOMEM || ret == -EINTR)
                        break;
 
                if (ret == -EBUSY || ret == -EINVAL) {
                        /* found lock contention or "pc" is obsolete. */
-                       busy = pc;
+                       busy = page;
                        cond_resched();
                } else
                        busy = NULL;
@@ -3662,14 +3656,14 @@ static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
  * make mem_cgroup's charge to be 0 if there is no task.
  * This enables deleting this mem_cgroup.
  */
-static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
+static int mem_cgroup_force_empty(struct mem_cgroup *memcg, bool free_all)
 {
        int ret;
        int node, zid, shrink;
        int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
-       struct cgroup *cgrp = mem->css.cgroup;
+       struct cgroup *cgrp = memcg->css.cgroup;
 
-       css_get(&mem->css);
+       css_get(&memcg->css);
 
        shrink = 0;
        /* should free all ? */
@@ -3685,14 +3679,14 @@ move_account:
                        goto out;
                /* This is for making all *used* pages to be on LRU. */
                lru_add_drain_all();
-               drain_all_stock_sync();
+               drain_all_stock_sync(memcg);
                ret = 0;
-               mem_cgroup_start_move(mem);
+               mem_cgroup_start_move(memcg);
                for_each_node_state(node, N_HIGH_MEMORY) {
                        for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
                                enum lru_list l;
                                for_each_lru(l) {
-                                       ret = mem_cgroup_force_empty_list(mem,
+                                       ret = mem_cgroup_force_empty_list(memcg,
                                                        node, zid, l);
                                        if (ret)
                                                break;
@@ -3701,16 +3695,16 @@ move_account:
                        if (ret)
                                break;
                }
-               mem_cgroup_end_move(mem);
-               memcg_oom_recover(mem);
+               mem_cgroup_end_move(memcg);
+               memcg_oom_recover(memcg);
                /* it seems parent cgroup doesn't have enough mem */
                if (ret == -ENOMEM)
                        goto try_to_free;
                cond_resched();
        /* "ret" should also be checked to ensure all lists are empty. */
-       } while (mem->res.usage > 0 || ret);
+       } while (memcg->res.usage > 0 || ret);
 out:
-       css_put(&mem->css);
+       css_put(&memcg->css);
        return ret;
 
 try_to_free:
@@ -3723,15 +3717,15 @@ try_to_free:
        lru_add_drain_all();
        /* try to free all pages in this cgroup */
        shrink = 1;
-       while (nr_retries && mem->res.usage > 0) {
+       while (nr_retries && memcg->res.usage > 0) {
                int progress;
 
                if (signal_pending(current)) {
                        ret = -EINTR;
                        goto out;
                }
-               progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
-                                               false, get_swappiness(mem));
+               progress = try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL,
+                                               false);
                if (!progress) {
                        nr_retries--;
                        /* maybe some writeback is necessary */
@@ -3759,12 +3753,12 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
                                        u64 val)
 {
        int retval = 0;
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
        struct cgroup *parent = cont->parent;
-       struct mem_cgroup *parent_mem = NULL;
+       struct mem_cgroup *parent_memcg = NULL;
 
        if (parent)
-               parent_mem = mem_cgroup_from_cont(parent);
+               parent_memcg = mem_cgroup_from_cont(parent);
 
        cgroup_lock();
        /*
@@ -3775,10 +3769,10 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
         * For the root cgroup, parent_mem is NULL, we allow value to be
         * set if there are no children.
         */
-       if ((!parent_mem || !parent_mem->use_hierarchy) &&
+       if ((!parent_memcg || !parent_memcg->use_hierarchy) &&
                                (val == 1 || val == 0)) {
                if (list_empty(&cont->children))
-                       mem->use_hierarchy = val;
+                       memcg->use_hierarchy = val;
                else
                        retval = -EBUSY;
        } else
@@ -3789,14 +3783,14 @@ static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
 }
 
 
-static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *mem,
+static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *memcg,
                                               enum mem_cgroup_stat_index idx)
 {
        struct mem_cgroup *iter;
        long val = 0;
 
        /* Per-cpu values can be negative, use a signed accumulator */
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                val += mem_cgroup_read_stat(iter, idx);
 
        if (val < 0) /* race ? */
@@ -3804,29 +3798,29 @@ static unsigned long mem_cgroup_recursive_stat(struct mem_cgroup *mem,
        return val;
 }
 
-static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
+static inline u64 mem_cgroup_usage(struct mem_cgroup *memcg, bool swap)
 {
        u64 val;
 
-       if (!mem_cgroup_is_root(mem)) {
+       if (!mem_cgroup_is_root(memcg)) {
                if (!swap)
-                       return res_counter_read_u64(&mem->res, RES_USAGE);
+                       return res_counter_read_u64(&memcg->res, RES_USAGE);
                else
-                       return res_counter_read_u64(&mem->memsw, RES_USAGE);
+                       return res_counter_read_u64(&memcg->memsw, RES_USAGE);
        }
 
-       val = mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_CACHE);
-       val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_RSS);
+       val = mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_CACHE);
+       val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_RSS);
 
        if (swap)
-               val += mem_cgroup_recursive_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
+               val += mem_cgroup_recursive_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
 
        return val << PAGE_SHIFT;
 }
 
 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
        u64 val;
        int type, name;
 
@@ -3835,15 +3829,15 @@ static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
        switch (type) {
        case _MEM:
                if (name == RES_USAGE)
-                       val = mem_cgroup_usage(mem, false);
+                       val = mem_cgroup_usage(memcg, false);
                else
-                       val = res_counter_read_u64(&mem->res, name);
+                       val = res_counter_read_u64(&memcg->res, name);
                break;
        case _MEMSWAP:
                if (name == RES_USAGE)
-                       val = mem_cgroup_usage(mem, true);
+                       val = mem_cgroup_usage(memcg, true);
                else
-                       val = res_counter_read_u64(&mem->memsw, name);
+                       val = res_counter_read_u64(&memcg->memsw, name);
                break;
        default:
                BUG();
@@ -3931,24 +3925,24 @@ out:
 
 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
 {
-       struct mem_cgroup *mem;
+       struct mem_cgroup *memcg;
        int type, name;
 
-       mem = mem_cgroup_from_cont(cont);
+       memcg = mem_cgroup_from_cont(cont);
        type = MEMFILE_TYPE(event);
        name = MEMFILE_ATTR(event);
        switch (name) {
        case RES_MAX_USAGE:
                if (type == _MEM)
-                       res_counter_reset_max(&mem->res);
+                       res_counter_reset_max(&memcg->res);
                else
-                       res_counter_reset_max(&mem->memsw);
+                       res_counter_reset_max(&memcg->memsw);
                break;
        case RES_FAILCNT:
                if (type == _MEM)
-                       res_counter_reset_failcnt(&mem->res);
+                       res_counter_reset_failcnt(&memcg->res);
                else
-                       res_counter_reset_failcnt(&mem->memsw);
+                       res_counter_reset_failcnt(&memcg->memsw);
                break;
        }
 
@@ -3965,7 +3959,7 @@ static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
                                        struct cftype *cft, u64 val)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
        if (val >= (1 << NR_MOVE_TYPE))
                return -EINVAL;
@@ -3975,7 +3969,7 @@ static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
         * inconsistent.
         */
        cgroup_lock();
-       mem->move_charge_at_immigrate = val;
+       memcg->move_charge_at_immigrate = val;
        cgroup_unlock();
 
        return 0;
@@ -4032,49 +4026,49 @@ struct {
 
 
 static void
-mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
+mem_cgroup_get_local_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 {
        s64 val;
 
        /* per cpu stat */
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_CACHE);
        s->stat[MCS_CACHE] += val * PAGE_SIZE;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_RSS);
        s->stat[MCS_RSS] += val * PAGE_SIZE;
-       val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
+       val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_FILE_MAPPED);
        s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
-       val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGIN);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGIN);
        s->stat[MCS_PGPGIN] += val;
-       val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGPGOUT);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGPGOUT);
        s->stat[MCS_PGPGOUT] += val;
        if (do_swap_account) {
-               val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
+               val = mem_cgroup_read_stat(memcg, MEM_CGROUP_STAT_SWAPOUT);
                s->stat[MCS_SWAP] += val * PAGE_SIZE;
        }
-       val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGFAULT);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGFAULT);
        s->stat[MCS_PGFAULT] += val;
-       val = mem_cgroup_read_events(mem, MEM_CGROUP_EVENTS_PGMAJFAULT);
+       val = mem_cgroup_read_events(memcg, MEM_CGROUP_EVENTS_PGMAJFAULT);
        s->stat[MCS_PGMAJFAULT] += val;
 
        /* per zone stat */
-       val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_ANON));
        s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_ANON));
        s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_INACTIVE_FILE));
        s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_ACTIVE_FILE));
        s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
-       val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
+       val = mem_cgroup_nr_lru_pages(memcg, BIT(LRU_UNEVICTABLE));
        s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
 }
 
 static void
-mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
+mem_cgroup_get_total_stat(struct mem_cgroup *memcg, struct mcs_total_stat *s)
 {
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                mem_cgroup_get_local_stat(iter, s);
 }
 
@@ -4087,35 +4081,37 @@ static int mem_control_numa_stat_show(struct seq_file *m, void *arg)
        struct cgroup *cont = m->private;
        struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
 
-       total_nr = mem_cgroup_nr_lru_pages(mem_cont);
+       total_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL);
        seq_printf(m, "total=%lu", total_nr);
        for_each_node_state(nid, N_HIGH_MEMORY) {
-               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid);
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid, LRU_ALL);
                seq_printf(m, " N%d=%lu", nid, node_nr);
        }
        seq_putc(m, '\n');
 
-       file_nr = mem_cgroup_nr_file_lru_pages(mem_cont);
+       file_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_FILE);
        seq_printf(m, "file=%lu", file_nr);
        for_each_node_state(nid, N_HIGH_MEMORY) {
-               node_nr = mem_cgroup_node_nr_file_lru_pages(mem_cont, nid);
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               LRU_ALL_FILE);
                seq_printf(m, " N%d=%lu", nid, node_nr);
        }
        seq_putc(m, '\n');
 
-       anon_nr = mem_cgroup_nr_anon_lru_pages(mem_cont);
+       anon_nr = mem_cgroup_nr_lru_pages(mem_cont, LRU_ALL_ANON);
        seq_printf(m, "anon=%lu", anon_nr);
        for_each_node_state(nid, N_HIGH_MEMORY) {
-               node_nr = mem_cgroup_node_nr_anon_lru_pages(mem_cont, nid);
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               LRU_ALL_ANON);
                seq_printf(m, " N%d=%lu", nid, node_nr);
        }
        seq_putc(m, '\n');
 
-       unevictable_nr = mem_cgroup_nr_unevictable_lru_pages(mem_cont);
+       unevictable_nr = mem_cgroup_nr_lru_pages(mem_cont, BIT(LRU_UNEVICTABLE));
        seq_printf(m, "unevictable=%lu", unevictable_nr);
        for_each_node_state(nid, N_HIGH_MEMORY) {
-               node_nr = mem_cgroup_node_nr_unevictable_lru_pages(mem_cont,
-                                                                       nid);
+               node_nr = mem_cgroup_node_nr_lru_pages(mem_cont, nid,
+                               BIT(LRU_UNEVICTABLE));
                seq_printf(m, " N%d=%lu", nid, node_nr);
        }
        seq_putc(m, '\n');
@@ -4158,8 +4154,6 @@ static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
        }
 
 #ifdef CONFIG_DEBUG_VM
-       cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
-
        {
                int nid, zid;
                struct mem_cgroup_per_zone *mz;
@@ -4193,7 +4187,7 @@ static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
 {
        struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
-       return get_swappiness(memcg);
+       return mem_cgroup_swappiness(memcg);
 }
 
 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
@@ -4296,20 +4290,20 @@ static int compare_thresholds(const void *a, const void *b)
        return _a->threshold - _b->threshold;
 }
 
-static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
+static int mem_cgroup_oom_notify_cb(struct mem_cgroup *memcg)
 {
        struct mem_cgroup_eventfd_list *ev;
 
-       list_for_each_entry(ev, &mem->oom_notify, list)
+       list_for_each_entry(ev, &memcg->oom_notify, list)
                eventfd_signal(ev->eventfd, 1);
        return 0;
 }
 
-static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
+static void mem_cgroup_oom_notify(struct mem_cgroup *memcg)
 {
        struct mem_cgroup *iter;
 
-       for_each_mem_cgroup_tree(iter, mem)
+       for_each_mem_cgroup_tree(iter, memcg)
                mem_cgroup_oom_notify_cb(iter);
 }
 
@@ -4483,15 +4477,15 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
        if (!event)
                return -ENOMEM;
 
-       mutex_lock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
 
        event->eventfd = eventfd;
        list_add(&event->list, &memcg->oom_notify);
 
        /* already in OOM ? */
-       if (atomic_read(&memcg->oom_lock))
+       if (atomic_read(&memcg->under_oom))
                eventfd_signal(eventfd, 1);
-       mutex_unlock(&memcg_oom_mutex);
+       spin_unlock(&memcg_oom_lock);
 
        return 0;
 }
@@ -4499,32 +4493,32 @@ static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
        struct cftype *cft, struct eventfd_ctx *eventfd)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
        struct mem_cgroup_eventfd_list *ev, *tmp;
        int type = MEMFILE_TYPE(cft->private);
 
        BUG_ON(type != _OOM_TYPE);
 
-       mutex_lock(&memcg_oom_mutex);
+       spin_lock(&memcg_oom_lock);
 
-       list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
+       list_for_each_entry_safe(ev, tmp, &memcg->oom_notify, list) {
                if (ev->eventfd == eventfd) {
                        list_del(&ev->list);
                        kfree(ev);
                }
        }
 
-       mutex_unlock(&memcg_oom_mutex);
+       spin_unlock(&memcg_oom_lock);
 }
 
 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
        struct cftype *cft,  struct cgroup_map_cb *cb)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
 
-       cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
+       cb->fill(cb, "oom_kill_disable", memcg->oom_kill_disable);
 
-       if (atomic_read(&mem->oom_lock))
+       if (atomic_read(&memcg->under_oom))
                cb->fill(cb, "under_oom", 1);
        else
                cb->fill(cb, "under_oom", 0);
@@ -4534,7 +4528,7 @@ static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
        struct cftype *cft, u64 val)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
        struct mem_cgroup *parent;
 
        /* cannot set to root cgroup and only 0 and 1 are allowed */
@@ -4546,13 +4540,13 @@ static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
        cgroup_lock();
        /* oom-kill-disable is a flag for subhierarchy. */
        if ((parent->use_hierarchy) ||
-           (mem->use_hierarchy && !list_empty(&cgrp->children))) {
+           (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
                cgroup_unlock();
                return -EINVAL;
        }
-       mem->oom_kill_disable = val;
+       memcg->oom_kill_disable = val;
        if (!val)
-               memcg_oom_recover(mem);
+               memcg_oom_recover(memcg);
        cgroup_unlock();
        return 0;
 }
@@ -4573,6 +4567,36 @@ static int mem_control_numa_stat_open(struct inode *unused, struct file *file)
 }
 #endif /* CONFIG_NUMA */
 
+#ifdef CONFIG_CGROUP_MEM_RES_CTLR_KMEM
+static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+{
+       /*
+        * Part of this would be better living in a separate allocation
+        * function, leaving us with just the cgroup tree population work.
+        * We, however, depend on state such as network's proto_list that
+        * is only initialized after cgroup creation. I found the less
+        * cumbersome way to deal with it to defer it all to populate time
+        */
+       return mem_cgroup_sockets_init(cont, ss);
+};
+
+static void kmem_cgroup_destroy(struct cgroup_subsys *ss,
+                               struct cgroup *cont)
+{
+       mem_cgroup_sockets_destroy(cont, ss);
+}
+#else
+static int register_kmem_files(struct cgroup *cont, struct cgroup_subsys *ss)
+{
+       return 0;
+}
+
+static void kmem_cgroup_destroy(struct cgroup_subsys *ss,
+                               struct cgroup *cont)
+{
+}
+#endif
+
 static struct cftype mem_cgroup_files[] = {
        {
                .name = "usage_in_bytes",
@@ -4688,7 +4712,7 @@ static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
 }
 #endif
 
-static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
        struct mem_cgroup_per_node *pn;
        struct mem_cgroup_per_zone *mz;
@@ -4708,21 +4732,21 @@ static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
        if (!pn)
                return 1;
 
-       mem->info.nodeinfo[node] = pn;
        for (zone = 0; zone < MAX_NR_ZONES; zone++) {
                mz = &pn->zoneinfo[zone];
                for_each_lru(l)
-                       INIT_LIST_HEAD(&mz->lists[l]);
+                       INIT_LIST_HEAD(&mz->lruvec.lists[l]);
                mz->usage_in_excess = 0;
                mz->on_tree = false;
-               mz->mem = mem;
+               mz->mem = memcg;
        }
+       memcg->info.nodeinfo[node] = pn;
        return 0;
 }
 
-static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
+static void free_mem_cgroup_per_zone_info(struct mem_cgroup *memcg, int node)
 {
-       kfree(mem->info.nodeinfo[node]);
+       kfree(memcg->info.nodeinfo[node]);
 }
 
 static struct mem_cgroup *mem_cgroup_alloc(void)
@@ -4764,52 +4788,53 @@ out_free:
  * Removal of cgroup itself succeeds regardless of refs from swap.
  */
 
-static void __mem_cgroup_free(struct mem_cgroup *mem)
+static void __mem_cgroup_free(struct mem_cgroup *memcg)
 {
        int node;
 
-       mem_cgroup_remove_from_trees(mem);
-       free_css_id(&mem_cgroup_subsys, &mem->css);
+       mem_cgroup_remove_from_trees(memcg);
+       free_css_id(&mem_cgroup_subsys, &memcg->css);
 
-       for_each_node_state(node, N_POSSIBLE)
-               free_mem_cgroup_per_zone_info(mem, node);
+       for_each_node(node)
+               free_mem_cgroup_per_zone_info(memcg, node);
 
-       free_percpu(mem->stat);
+       free_percpu(memcg->stat);
        if (sizeof(struct mem_cgroup) < PAGE_SIZE)
-               kfree(mem);
+               kfree(memcg);
        else
-               vfree(mem);
+               vfree(memcg);
 }
 
-static void mem_cgroup_get(struct mem_cgroup *mem)
+static void mem_cgroup_get(struct mem_cgroup *memcg)
 {
-       atomic_inc(&mem->refcnt);
+       atomic_inc(&memcg->refcnt);
 }
 
-static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
+static void __mem_cgroup_put(struct mem_cgroup *memcg, int count)
 {
-       if (atomic_sub_and_test(count, &mem->refcnt)) {
-               struct mem_cgroup *parent = parent_mem_cgroup(mem);
-               __mem_cgroup_free(mem);
+       if (atomic_sub_and_test(count, &memcg->refcnt)) {
+               struct mem_cgroup *parent = parent_mem_cgroup(memcg);
+               __mem_cgroup_free(memcg);
                if (parent)
                        mem_cgroup_put(parent);
        }
 }
 
-static void mem_cgroup_put(struct mem_cgroup *mem)
+static void mem_cgroup_put(struct mem_cgroup *memcg)
 {
-       __mem_cgroup_put(mem, 1);
+       __mem_cgroup_put(memcg, 1);
 }
 
 /*
  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
  */
-static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
+struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *memcg)
 {
-       if (!mem->res.parent)
+       if (!memcg->res.parent)
                return NULL;
-       return mem_cgroup_from_res_counter(mem->res.parent, res);
+       return mem_cgroup_from_res_counter(memcg->res.parent, res);
 }
+EXPORT_SYMBOL(parent_mem_cgroup);
 
 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
 static void __init enable_swap_cgroup(void)
@@ -4829,13 +4854,13 @@ static int mem_cgroup_soft_limit_tree_init(void)
        struct mem_cgroup_tree_per_zone *rtpz;
        int tmp, node, zone;
 
-       for_each_node_state(node, N_POSSIBLE) {
+       for_each_node(node) {
                tmp = node;
                if (!node_state(node, N_NORMAL_MEMORY))
                        tmp = -1;
                rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
                if (!rtpn)
-                       return 1;
+                       goto err_cleanup;
 
                soft_limit_tree.rb_tree_per_node[node] = rtpn;
 
@@ -4846,21 +4871,31 @@ static int mem_cgroup_soft_limit_tree_init(void)
                }
        }
        return 0;
+
+err_cleanup:
+       for_each_node(node) {
+               if (!soft_limit_tree.rb_tree_per_node[node])
+                       break;
+               kfree(soft_limit_tree.rb_tree_per_node[node]);
+               soft_limit_tree.rb_tree_per_node[node] = NULL;
+       }
+       return 1;
+
 }
 
 static struct cgroup_subsys_state * __ref
 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
 {
-       struct mem_cgroup *mem, *parent;
+       struct mem_cgroup *memcg, *parent;
        long error = -ENOMEM;
        int node;
 
-       mem = mem_cgroup_alloc();
-       if (!mem)
+       memcg = mem_cgroup_alloc();
+       if (!memcg)
                return ERR_PTR(error);
 
-       for_each_node_state(node, N_POSSIBLE)
-               if (alloc_mem_cgroup_per_zone_info(mem, node))
+       for_each_node(node)
+               if (alloc_mem_cgroup_per_zone_info(memcg, node))
                        goto free_out;
 
        /* root ? */
@@ -4868,9 +4903,9 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                int cpu;
                enable_swap_cgroup();
                parent = NULL;
-               root_mem_cgroup = mem;
                if (mem_cgroup_soft_limit_tree_init())
                        goto free_out;
+               root_mem_cgroup = memcg;
                for_each_possible_cpu(cpu) {
                        struct memcg_stock_pcp *stock =
                                                &per_cpu(memcg_stock, cpu);
@@ -4879,13 +4914,13 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
        } else {
                parent = mem_cgroup_from_cont(cont->parent);
-               mem->use_hierarchy = parent->use_hierarchy;
-               mem->oom_kill_disable = parent->oom_kill_disable;
+               memcg->use_hierarchy = parent->use_hierarchy;
+               memcg->oom_kill_disable = parent->oom_kill_disable;
        }
 
        if (parent && parent->use_hierarchy) {
-               res_counter_init(&mem->res, &parent->res);
-               res_counter_init(&mem->memsw, &parent->memsw);
+               res_counter_init(&memcg->res, &parent->res);
+               res_counter_init(&memcg->memsw, &parent->memsw);
                /*
                 * We increment refcnt of the parent to ensure that we can
                 * safely access it on res_counter_charge/uncharge.
@@ -4894,39 +4929,39 @@ mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
                 */
                mem_cgroup_get(parent);
        } else {
-               res_counter_init(&mem->res, NULL);
-               res_counter_init(&mem->memsw, NULL);
+               res_counter_init(&memcg->res, NULL);
+               res_counter_init(&memcg->memsw, NULL);
        }
-       mem->last_scanned_child = 0;
-       mem->last_scanned_node = MAX_NUMNODES;
-       INIT_LIST_HEAD(&mem->oom_notify);
+       memcg->last_scanned_node = MAX_NUMNODES;
+       INIT_LIST_HEAD(&memcg->oom_notify);
 
        if (parent)
-               mem->swappiness = get_swappiness(parent);
-       atomic_set(&mem->refcnt, 1);
-       mem->move_charge_at_immigrate = 0;
-       mutex_init(&mem->thresholds_lock);
-       return &mem->css;
+               memcg->swappiness = mem_cgroup_swappiness(parent);
+       atomic_set(&memcg->refcnt, 1);
+       memcg->move_charge_at_immigrate = 0;
+       mutex_init(&memcg->thresholds_lock);
+       return &memcg->css;
 free_out:
-       __mem_cgroup_free(mem);
-       root_mem_cgroup = NULL;
+       __mem_cgroup_free(memcg);
        return ERR_PTR(error);
 }
 
 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
                                        struct cgroup *cont)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
 
-       return mem_cgroup_force_empty(mem, false);
+       return mem_cgroup_force_empty(memcg, false);
 }
 
 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
                                struct cgroup *cont)
 {
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
+
+       kmem_cgroup_destroy(ss, cont);
 
-       mem_cgroup_put(mem);
+       mem_cgroup_put(memcg);
 }
 
 static int mem_cgroup_populate(struct cgroup_subsys *ss,
@@ -4939,6 +4974,10 @@ static int mem_cgroup_populate(struct cgroup_subsys *ss,
 
        if (!ret)
                ret = register_memsw_files(cont, ss);
+
+       if (!ret)
+               ret = register_kmem_files(cont, ss);
+
        return ret;
 }
 
@@ -4949,9 +4988,9 @@ static int mem_cgroup_do_precharge(unsigned long count)
 {
        int ret = 0;
        int batch_count = PRECHARGE_COUNT_AT_ONCE;
-       struct mem_cgroup *mem = mc.to;
+       struct mem_cgroup *memcg = mc.to;
 
-       if (mem_cgroup_is_root(mem)) {
+       if (mem_cgroup_is_root(memcg)) {
                mc.precharge += count;
                /* we don't need css_get for root */
                return ret;
@@ -4960,16 +4999,16 @@ static int mem_cgroup_do_precharge(unsigned long count)
        if (count > 1) {
                struct res_counter *dummy;
                /*
-                * "mem" cannot be under rmdir() because we've already checked
+                * "memcg" cannot be under rmdir() because we've already checked
                 * by cgroup_lock_live_cgroup() that it is not removed and we
                 * are still under the same cgroup_mutex. So we can postpone
                 * css_get().
                 */
-               if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
+               if (res_counter_charge(&memcg->res, PAGE_SIZE * count, &dummy))
                        goto one_by_one;
-               if (do_swap_account && res_counter_charge(&mem->memsw,
+               if (do_swap_account && res_counter_charge(&memcg->memsw,
                                                PAGE_SIZE * count, &dummy)) {
-                       res_counter_uncharge(&mem->res, PAGE_SIZE * count);
+                       res_counter_uncharge(&memcg->res, PAGE_SIZE * count);
                        goto one_by_one;
                }
                mc.precharge += count;
@@ -4986,10 +5025,11 @@ one_by_one:
                        batch_count = PRECHARGE_COUNT_AT_ONCE;
                        cond_resched();
                }
-               ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, 1, &mem, false);
-               if (ret || !mem)
+               ret = __mem_cgroup_try_charge(NULL,
+                                       GFP_KERNEL, 1, &memcg, false);
+               if (ret)
                        /* mem_cgroup_clear_mc() will do uncharge later */
-                       return -ENOMEM;
+                       return ret;
                mc.precharge++;
        }
        return ret;
@@ -5086,15 +5126,17 @@ static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
                pgoff = pte_to_pgoff(ptent);
 
        /* page is moved even if it's not RSS of this task(page-faulted). */
-       if (!mapping_cap_swap_backed(mapping)) { /* normal file */
-               page = find_get_page(mapping, pgoff);
-       } else { /* shmem/tmpfs file. we should take account of swap too. */
-               swp_entry_t ent;
-               mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
+       page = find_get_page(mapping, pgoff);
+
+#ifdef CONFIG_SWAP
+       /* shmem/tmpfs may report page out on swap: account for that too. */
+       if (radix_tree_exceptional_entry(page)) {
+               swp_entry_t swap = radix_to_swp_entry(page);
                if (do_swap_account)
-                       entry->val = ent.val;
+                       *entry = swap;
+               page = find_get_page(&swapper_space, swap.val);
        }
-
+#endif
        return page;
 }
 
@@ -5132,7 +5174,7 @@ static int is_target_pte_for_mc(struct vm_area_struct *vma,
        }
        /* There is a swap entry and a page doesn't exist or isn't charged */
        if (ent.val && !ret &&
-                       css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
+                       css_id(&mc.from->css) == lookup_swap_cgroup_id(ent)) {
                ret = MC_TARGET_SWAP;
                if (target)
                        target->ent = ent;
@@ -5256,16 +5298,17 @@ static void mem_cgroup_clear_mc(void)
 
 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
+       struct task_struct *p = cgroup_taskset_first(tset);
        int ret = 0;
-       struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
+       struct mem_cgroup *memcg = mem_cgroup_from_cont(cgroup);
 
-       if (mem->move_charge_at_immigrate) {
+       if (memcg->move_charge_at_immigrate) {
                struct mm_struct *mm;
                struct mem_cgroup *from = mem_cgroup_from_task(p);
 
-               VM_BUG_ON(from == mem);
+               VM_BUG_ON(from == memcg);
 
                mm = get_task_mm(p);
                if (!mm)
@@ -5280,7 +5323,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                        mem_cgroup_start_move(from);
                        spin_lock(&mc.lock);
                        mc.from = from;
-                       mc.to = mem;
+                       mc.to = memcg;
                        spin_unlock(&mc.lock);
                        /* We set mc.moving_task later */
 
@@ -5295,7 +5338,7 @@ static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
 
 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
        mem_cgroup_clear_mc();
 }
@@ -5412,9 +5455,9 @@ retry:
 
 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                struct cgroup *cont,
-                               struct cgroup *old_cont,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
+       struct task_struct *p = cgroup_taskset_first(tset);
        struct mm_struct *mm = get_task_mm(p);
 
        if (mm) {
@@ -5429,19 +5472,18 @@ static void mem_cgroup_move_task(struct cgroup_subsys *ss,
 #else  /* !CONFIG_MMU */
 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
        return 0;
 }
 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
                                struct cgroup *cgroup,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
 }
 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
                                struct cgroup *cont,
-                               struct cgroup *old_cont,
-                               struct task_struct *p)
+                               struct cgroup_taskset *tset)
 {
 }
 #endif