memcg: fix wrong VM_BUG_ON() in try_charge()'s mm->owner check
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  */
23
24 #include <linux/res_counter.h>
25 #include <linux/memcontrol.h>
26 #include <linux/cgroup.h>
27 #include <linux/mm.h>
28 #include <linux/hugetlb.h>
29 #include <linux/pagemap.h>
30 #include <linux/smp.h>
31 #include <linux/page-flags.h>
32 #include <linux/backing-dev.h>
33 #include <linux/bit_spinlock.h>
34 #include <linux/rcupdate.h>
35 #include <linux/limits.h>
36 #include <linux/mutex.h>
37 #include <linux/rbtree.h>
38 #include <linux/slab.h>
39 #include <linux/swap.h>
40 #include <linux/swapops.h>
41 #include <linux/spinlock.h>
42 #include <linux/eventfd.h>
43 #include <linux/sort.h>
44 #include <linux/fs.h>
45 #include <linux/seq_file.h>
46 #include <linux/vmalloc.h>
47 #include <linux/mm_inline.h>
48 #include <linux/page_cgroup.h>
49 #include <linux/cpu.h>
50 #include <linux/oom.h>
51 #include "internal.h"
52
53 #include <asm/uaccess.h>
54
55 #include <trace/events/vmscan.h>
56
57 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
58 #define MEM_CGROUP_RECLAIM_RETRIES      5
59 struct mem_cgroup *root_mem_cgroup __read_mostly;
60
61 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
62 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
63 int do_swap_account __read_mostly;
64
65 /* for remember boot option*/
66 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
67 static int really_do_swap_account __initdata = 1;
68 #else
69 static int really_do_swap_account __initdata = 0;
70 #endif
71
72 #else
73 #define do_swap_account         (0)
74 #endif
75
76 /*
77  * Per memcg event counter is incremented at every pagein/pageout. This counter
78  * is used for trigger some periodic events. This is straightforward and better
79  * than using jiffies etc. to handle periodic memcg event.
80  *
81  * These values will be used as !((event) & ((1 <<(thresh)) - 1))
82  */
83 #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
84 #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
85
86 /*
87  * Statistics for memory cgroup.
88  */
89 enum mem_cgroup_stat_index {
90         /*
91          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
92          */
93         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
94         MEM_CGROUP_STAT_RSS,       /* # of pages charged as anon rss */
95         MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
96         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
97         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
98         MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
99         MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
100         /* incremented at every  pagein/pageout */
101         MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
102         MEM_CGROUP_ON_MOVE,     /* someone is moving account between groups */
103
104         MEM_CGROUP_STAT_NSTATS,
105 };
106
107 struct mem_cgroup_stat_cpu {
108         s64 count[MEM_CGROUP_STAT_NSTATS];
109 };
110
111 /*
112  * per-zone information in memory controller.
113  */
114 struct mem_cgroup_per_zone {
115         /*
116          * spin_lock to protect the per cgroup LRU
117          */
118         struct list_head        lists[NR_LRU_LISTS];
119         unsigned long           count[NR_LRU_LISTS];
120
121         struct zone_reclaim_stat reclaim_stat;
122         struct rb_node          tree_node;      /* RB tree node */
123         unsigned long long      usage_in_excess;/* Set to the value by which */
124                                                 /* the soft limit is exceeded*/
125         bool                    on_tree;
126         struct mem_cgroup       *mem;           /* Back pointer, we cannot */
127                                                 /* use container_of        */
128 };
129 /* Macro for accessing counter */
130 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
131
132 struct mem_cgroup_per_node {
133         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
134 };
135
136 struct mem_cgroup_lru_info {
137         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
138 };
139
140 /*
141  * Cgroups above their limits are maintained in a RB-Tree, independent of
142  * their hierarchy representation
143  */
144
145 struct mem_cgroup_tree_per_zone {
146         struct rb_root rb_root;
147         spinlock_t lock;
148 };
149
150 struct mem_cgroup_tree_per_node {
151         struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
152 };
153
154 struct mem_cgroup_tree {
155         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
156 };
157
158 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
159
160 struct mem_cgroup_threshold {
161         struct eventfd_ctx *eventfd;
162         u64 threshold;
163 };
164
165 /* For threshold */
166 struct mem_cgroup_threshold_ary {
167         /* An array index points to threshold just below usage. */
168         int current_threshold;
169         /* Size of entries[] */
170         unsigned int size;
171         /* Array of thresholds */
172         struct mem_cgroup_threshold entries[0];
173 };
174
175 struct mem_cgroup_thresholds {
176         /* Primary thresholds array */
177         struct mem_cgroup_threshold_ary *primary;
178         /*
179          * Spare threshold array.
180          * This is needed to make mem_cgroup_unregister_event() "never fail".
181          * It must be able to store at least primary->size - 1 entries.
182          */
183         struct mem_cgroup_threshold_ary *spare;
184 };
185
186 /* for OOM */
187 struct mem_cgroup_eventfd_list {
188         struct list_head list;
189         struct eventfd_ctx *eventfd;
190 };
191
192 static void mem_cgroup_threshold(struct mem_cgroup *mem);
193 static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
194
195 /*
196  * The memory controller data structure. The memory controller controls both
197  * page cache and RSS per cgroup. We would eventually like to provide
198  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
199  * to help the administrator determine what knobs to tune.
200  *
201  * TODO: Add a water mark for the memory controller. Reclaim will begin when
202  * we hit the water mark. May be even add a low water mark, such that
203  * no reclaim occurs from a cgroup at it's low water mark, this is
204  * a feature that will be implemented much later in the future.
205  */
206 struct mem_cgroup {
207         struct cgroup_subsys_state css;
208         /*
209          * the counter to account for memory usage
210          */
211         struct res_counter res;
212         /*
213          * the counter to account for mem+swap usage.
214          */
215         struct res_counter memsw;
216         /*
217          * Per cgroup active and inactive list, similar to the
218          * per zone LRU lists.
219          */
220         struct mem_cgroup_lru_info info;
221
222         /*
223           protect against reclaim related member.
224         */
225         spinlock_t reclaim_param_lock;
226
227         /*
228          * While reclaiming in a hierarchy, we cache the last child we
229          * reclaimed from.
230          */
231         int last_scanned_child;
232         /*
233          * Should the accounting and control be hierarchical, per subtree?
234          */
235         bool use_hierarchy;
236         atomic_t        oom_lock;
237         atomic_t        refcnt;
238
239         unsigned int    swappiness;
240         /* OOM-Killer disable */
241         int             oom_kill_disable;
242
243         /* set when res.limit == memsw.limit */
244         bool            memsw_is_minimum;
245
246         /* protect arrays of thresholds */
247         struct mutex thresholds_lock;
248
249         /* thresholds for memory usage. RCU-protected */
250         struct mem_cgroup_thresholds thresholds;
251
252         /* thresholds for mem+swap usage. RCU-protected */
253         struct mem_cgroup_thresholds memsw_thresholds;
254
255         /* For oom notifier event fd */
256         struct list_head oom_notify;
257
258         /*
259          * Should we move charges of a task when a task is moved into this
260          * mem_cgroup ? And what type of charges should we move ?
261          */
262         unsigned long   move_charge_at_immigrate;
263         /*
264          * percpu counter.
265          */
266         struct mem_cgroup_stat_cpu *stat;
267         /*
268          * used when a cpu is offlined or other synchronizations
269          * See mem_cgroup_read_stat().
270          */
271         struct mem_cgroup_stat_cpu nocpu_base;
272         spinlock_t pcp_counter_lock;
273 };
274
275 /* Stuffs for move charges at task migration. */
276 /*
277  * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
278  * left-shifted bitmap of these types.
279  */
280 enum move_type {
281         MOVE_CHARGE_TYPE_ANON,  /* private anonymous page and swap of it */
282         MOVE_CHARGE_TYPE_FILE,  /* file page(including tmpfs) and swap of it */
283         NR_MOVE_TYPE,
284 };
285
286 /* "mc" and its members are protected by cgroup_mutex */
287 static struct move_charge_struct {
288         spinlock_t        lock; /* for from, to */
289         struct mem_cgroup *from;
290         struct mem_cgroup *to;
291         unsigned long precharge;
292         unsigned long moved_charge;
293         unsigned long moved_swap;
294         struct task_struct *moving_task;        /* a task moving charges */
295         struct mm_struct *mm;
296         wait_queue_head_t waitq;                /* a waitq for other context */
297 } mc = {
298         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
299         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
300 };
301
302 static bool move_anon(void)
303 {
304         return test_bit(MOVE_CHARGE_TYPE_ANON,
305                                         &mc.to->move_charge_at_immigrate);
306 }
307
308 static bool move_file(void)
309 {
310         return test_bit(MOVE_CHARGE_TYPE_FILE,
311                                         &mc.to->move_charge_at_immigrate);
312 }
313
314 /*
315  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
316  * limit reclaim to prevent infinite loops, if they ever occur.
317  */
318 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            (100)
319 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
320
321 enum charge_type {
322         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
323         MEM_CGROUP_CHARGE_TYPE_MAPPED,
324         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
325         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
326         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
327         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
328         NR_CHARGE_TYPE,
329 };
330
331 /* only for here (for easy reading.) */
332 #define PCGF_CACHE      (1UL << PCG_CACHE)
333 #define PCGF_USED       (1UL << PCG_USED)
334 #define PCGF_LOCK       (1UL << PCG_LOCK)
335 /* Not used, but added here for completeness */
336 #define PCGF_ACCT       (1UL << PCG_ACCT)
337
338 /* for encoding cft->private value on file */
339 #define _MEM                    (0)
340 #define _MEMSWAP                (1)
341 #define _OOM_TYPE               (2)
342 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
343 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
344 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
345 /* Used for OOM nofiier */
346 #define OOM_CONTROL             (0)
347
348 /*
349  * Reclaim flags for mem_cgroup_hierarchical_reclaim
350  */
351 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
352 #define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
353 #define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
354 #define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
355 #define MEM_CGROUP_RECLAIM_SOFT_BIT     0x2
356 #define MEM_CGROUP_RECLAIM_SOFT         (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
357
358 static void mem_cgroup_get(struct mem_cgroup *mem);
359 static void mem_cgroup_put(struct mem_cgroup *mem);
360 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
361 static void drain_all_stock_async(void);
362
363 static struct mem_cgroup_per_zone *
364 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
365 {
366         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
367 }
368
369 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
370 {
371         return &mem->css;
372 }
373
374 static struct mem_cgroup_per_zone *
375 page_cgroup_zoneinfo(struct page_cgroup *pc)
376 {
377         struct mem_cgroup *mem = pc->mem_cgroup;
378         int nid = page_cgroup_nid(pc);
379         int zid = page_cgroup_zid(pc);
380
381         if (!mem)
382                 return NULL;
383
384         return mem_cgroup_zoneinfo(mem, nid, zid);
385 }
386
387 static struct mem_cgroup_tree_per_zone *
388 soft_limit_tree_node_zone(int nid, int zid)
389 {
390         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
391 }
392
393 static struct mem_cgroup_tree_per_zone *
394 soft_limit_tree_from_page(struct page *page)
395 {
396         int nid = page_to_nid(page);
397         int zid = page_zonenum(page);
398
399         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
400 }
401
402 static void
403 __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
404                                 struct mem_cgroup_per_zone *mz,
405                                 struct mem_cgroup_tree_per_zone *mctz,
406                                 unsigned long long new_usage_in_excess)
407 {
408         struct rb_node **p = &mctz->rb_root.rb_node;
409         struct rb_node *parent = NULL;
410         struct mem_cgroup_per_zone *mz_node;
411
412         if (mz->on_tree)
413                 return;
414
415         mz->usage_in_excess = new_usage_in_excess;
416         if (!mz->usage_in_excess)
417                 return;
418         while (*p) {
419                 parent = *p;
420                 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
421                                         tree_node);
422                 if (mz->usage_in_excess < mz_node->usage_in_excess)
423                         p = &(*p)->rb_left;
424                 /*
425                  * We can't avoid mem cgroups that are over their soft
426                  * limit by the same amount
427                  */
428                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
429                         p = &(*p)->rb_right;
430         }
431         rb_link_node(&mz->tree_node, parent, p);
432         rb_insert_color(&mz->tree_node, &mctz->rb_root);
433         mz->on_tree = true;
434 }
435
436 static void
437 __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
438                                 struct mem_cgroup_per_zone *mz,
439                                 struct mem_cgroup_tree_per_zone *mctz)
440 {
441         if (!mz->on_tree)
442                 return;
443         rb_erase(&mz->tree_node, &mctz->rb_root);
444         mz->on_tree = false;
445 }
446
447 static void
448 mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
449                                 struct mem_cgroup_per_zone *mz,
450                                 struct mem_cgroup_tree_per_zone *mctz)
451 {
452         spin_lock(&mctz->lock);
453         __mem_cgroup_remove_exceeded(mem, mz, mctz);
454         spin_unlock(&mctz->lock);
455 }
456
457
458 static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
459 {
460         unsigned long long excess;
461         struct mem_cgroup_per_zone *mz;
462         struct mem_cgroup_tree_per_zone *mctz;
463         int nid = page_to_nid(page);
464         int zid = page_zonenum(page);
465         mctz = soft_limit_tree_from_page(page);
466
467         /*
468          * Necessary to update all ancestors when hierarchy is used.
469          * because their event counter is not touched.
470          */
471         for (; mem; mem = parent_mem_cgroup(mem)) {
472                 mz = mem_cgroup_zoneinfo(mem, nid, zid);
473                 excess = res_counter_soft_limit_excess(&mem->res);
474                 /*
475                  * We have to update the tree if mz is on RB-tree or
476                  * mem is over its softlimit.
477                  */
478                 if (excess || mz->on_tree) {
479                         spin_lock(&mctz->lock);
480                         /* if on-tree, remove it */
481                         if (mz->on_tree)
482                                 __mem_cgroup_remove_exceeded(mem, mz, mctz);
483                         /*
484                          * Insert again. mz->usage_in_excess will be updated.
485                          * If excess is 0, no tree ops.
486                          */
487                         __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
488                         spin_unlock(&mctz->lock);
489                 }
490         }
491 }
492
493 static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
494 {
495         int node, zone;
496         struct mem_cgroup_per_zone *mz;
497         struct mem_cgroup_tree_per_zone *mctz;
498
499         for_each_node_state(node, N_POSSIBLE) {
500                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
501                         mz = mem_cgroup_zoneinfo(mem, node, zone);
502                         mctz = soft_limit_tree_node_zone(node, zone);
503                         mem_cgroup_remove_exceeded(mem, mz, mctz);
504                 }
505         }
506 }
507
508 static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
509 {
510         return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
511 }
512
513 static struct mem_cgroup_per_zone *
514 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
515 {
516         struct rb_node *rightmost = NULL;
517         struct mem_cgroup_per_zone *mz;
518
519 retry:
520         mz = NULL;
521         rightmost = rb_last(&mctz->rb_root);
522         if (!rightmost)
523                 goto done;              /* Nothing to reclaim from */
524
525         mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
526         /*
527          * Remove the node now but someone else can add it back,
528          * we will to add it back at the end of reclaim to its correct
529          * position in the tree.
530          */
531         __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
532         if (!res_counter_soft_limit_excess(&mz->mem->res) ||
533                 !css_tryget(&mz->mem->css))
534                 goto retry;
535 done:
536         return mz;
537 }
538
539 static struct mem_cgroup_per_zone *
540 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
541 {
542         struct mem_cgroup_per_zone *mz;
543
544         spin_lock(&mctz->lock);
545         mz = __mem_cgroup_largest_soft_limit_node(mctz);
546         spin_unlock(&mctz->lock);
547         return mz;
548 }
549
550 /*
551  * Implementation Note: reading percpu statistics for memcg.
552  *
553  * Both of vmstat[] and percpu_counter has threshold and do periodic
554  * synchronization to implement "quick" read. There are trade-off between
555  * reading cost and precision of value. Then, we may have a chance to implement
556  * a periodic synchronizion of counter in memcg's counter.
557  *
558  * But this _read() function is used for user interface now. The user accounts
559  * memory usage by memory cgroup and he _always_ requires exact value because
560  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
561  * have to visit all online cpus and make sum. So, for now, unnecessary
562  * synchronization is not implemented. (just implemented for cpu hotplug)
563  *
564  * If there are kernel internal actions which can make use of some not-exact
565  * value, and reading all cpu value can be performance bottleneck in some
566  * common workload, threashold and synchonization as vmstat[] should be
567  * implemented.
568  */
569 static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
570                 enum mem_cgroup_stat_index idx)
571 {
572         int cpu;
573         s64 val = 0;
574
575         get_online_cpus();
576         for_each_online_cpu(cpu)
577                 val += per_cpu(mem->stat->count[idx], cpu);
578 #ifdef CONFIG_HOTPLUG_CPU
579         spin_lock(&mem->pcp_counter_lock);
580         val += mem->nocpu_base.count[idx];
581         spin_unlock(&mem->pcp_counter_lock);
582 #endif
583         put_online_cpus();
584         return val;
585 }
586
587 static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
588 {
589         s64 ret;
590
591         ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
592         ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
593         return ret;
594 }
595
596 static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
597                                          bool charge)
598 {
599         int val = (charge) ? 1 : -1;
600         this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
601 }
602
603 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
604                                          struct page_cgroup *pc,
605                                          bool charge)
606 {
607         int val = (charge) ? 1 : -1;
608
609         preempt_disable();
610
611         if (PageCgroupCache(pc))
612                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
613         else
614                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
615
616         if (charge)
617                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
618         else
619                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
620         __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
621
622         preempt_enable();
623 }
624
625 static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
626                                         enum lru_list idx)
627 {
628         int nid, zid;
629         struct mem_cgroup_per_zone *mz;
630         u64 total = 0;
631
632         for_each_online_node(nid)
633                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
634                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
635                         total += MEM_CGROUP_ZSTAT(mz, idx);
636                 }
637         return total;
638 }
639
640 static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
641 {
642         s64 val;
643
644         val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
645
646         return !(val & ((1 << event_mask_shift) - 1));
647 }
648
649 /*
650  * Check events in order.
651  *
652  */
653 static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
654 {
655         /* threshold event is triggered in finer grain than soft limit */
656         if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
657                 mem_cgroup_threshold(mem);
658                 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
659                         mem_cgroup_update_tree(mem, page);
660         }
661 }
662
663 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
664 {
665         return container_of(cgroup_subsys_state(cont,
666                                 mem_cgroup_subsys_id), struct mem_cgroup,
667                                 css);
668 }
669
670 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
671 {
672         /*
673          * mm_update_next_owner() may clear mm->owner to NULL
674          * if it races with swapoff, page migration, etc.
675          * So this can be called with p == NULL.
676          */
677         if (unlikely(!p))
678                 return NULL;
679
680         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
681                                 struct mem_cgroup, css);
682 }
683
684 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
685 {
686         struct mem_cgroup *mem = NULL;
687
688         if (!mm)
689                 return NULL;
690         /*
691          * Because we have no locks, mm->owner's may be being moved to other
692          * cgroup. We use css_tryget() here even if this looks
693          * pessimistic (rather than adding locks here).
694          */
695         rcu_read_lock();
696         do {
697                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
698                 if (unlikely(!mem))
699                         break;
700         } while (!css_tryget(&mem->css));
701         rcu_read_unlock();
702         return mem;
703 }
704
705 /* The caller has to guarantee "mem" exists before calling this */
706 static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
707 {
708         struct cgroup_subsys_state *css;
709         int found;
710
711         if (!mem) /* ROOT cgroup has the smallest ID */
712                 return root_mem_cgroup; /*css_put/get against root is ignored*/
713         if (!mem->use_hierarchy) {
714                 if (css_tryget(&mem->css))
715                         return mem;
716                 return NULL;
717         }
718         rcu_read_lock();
719         /*
720          * searching a memory cgroup which has the smallest ID under given
721          * ROOT cgroup. (ID >= 1)
722          */
723         css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
724         if (css && css_tryget(css))
725                 mem = container_of(css, struct mem_cgroup, css);
726         else
727                 mem = NULL;
728         rcu_read_unlock();
729         return mem;
730 }
731
732 static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
733                                         struct mem_cgroup *root,
734                                         bool cond)
735 {
736         int nextid = css_id(&iter->css) + 1;
737         int found;
738         int hierarchy_used;
739         struct cgroup_subsys_state *css;
740
741         hierarchy_used = iter->use_hierarchy;
742
743         css_put(&iter->css);
744         /* If no ROOT, walk all, ignore hierarchy */
745         if (!cond || (root && !hierarchy_used))
746                 return NULL;
747
748         if (!root)
749                 root = root_mem_cgroup;
750
751         do {
752                 iter = NULL;
753                 rcu_read_lock();
754
755                 css = css_get_next(&mem_cgroup_subsys, nextid,
756                                 &root->css, &found);
757                 if (css && css_tryget(css))
758                         iter = container_of(css, struct mem_cgroup, css);
759                 rcu_read_unlock();
760                 /* If css is NULL, no more cgroups will be found */
761                 nextid = found + 1;
762         } while (css && !iter);
763
764         return iter;
765 }
766 /*
767  * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
768  * be careful that "break" loop is not allowed. We have reference count.
769  * Instead of that modify "cond" to be false and "continue" to exit the loop.
770  */
771 #define for_each_mem_cgroup_tree_cond(iter, root, cond) \
772         for (iter = mem_cgroup_start_loop(root);\
773              iter != NULL;\
774              iter = mem_cgroup_get_next(iter, root, cond))
775
776 #define for_each_mem_cgroup_tree(iter, root) \
777         for_each_mem_cgroup_tree_cond(iter, root, true)
778
779 #define for_each_mem_cgroup_all(iter) \
780         for_each_mem_cgroup_tree_cond(iter, NULL, true)
781
782
783 static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
784 {
785         return (mem == root_mem_cgroup);
786 }
787
788 /*
789  * Following LRU functions are allowed to be used without PCG_LOCK.
790  * Operations are called by routine of global LRU independently from memcg.
791  * What we have to take care of here is validness of pc->mem_cgroup.
792  *
793  * Changes to pc->mem_cgroup happens when
794  * 1. charge
795  * 2. moving account
796  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
797  * It is added to LRU before charge.
798  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
799  * When moving account, the page is not on LRU. It's isolated.
800  */
801
802 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
803 {
804         struct page_cgroup *pc;
805         struct mem_cgroup_per_zone *mz;
806
807         if (mem_cgroup_disabled())
808                 return;
809         pc = lookup_page_cgroup(page);
810         /* can happen while we handle swapcache. */
811         if (!TestClearPageCgroupAcctLRU(pc))
812                 return;
813         VM_BUG_ON(!pc->mem_cgroup);
814         /*
815          * We don't check PCG_USED bit. It's cleared when the "page" is finally
816          * removed from global LRU.
817          */
818         mz = page_cgroup_zoneinfo(pc);
819         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
820         if (mem_cgroup_is_root(pc->mem_cgroup))
821                 return;
822         VM_BUG_ON(list_empty(&pc->lru));
823         list_del_init(&pc->lru);
824         return;
825 }
826
827 void mem_cgroup_del_lru(struct page *page)
828 {
829         mem_cgroup_del_lru_list(page, page_lru(page));
830 }
831
832 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
833 {
834         struct mem_cgroup_per_zone *mz;
835         struct page_cgroup *pc;
836
837         if (mem_cgroup_disabled())
838                 return;
839
840         pc = lookup_page_cgroup(page);
841         /*
842          * Used bit is set without atomic ops but after smp_wmb().
843          * For making pc->mem_cgroup visible, insert smp_rmb() here.
844          */
845         smp_rmb();
846         /* unused or root page is not rotated. */
847         if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
848                 return;
849         mz = page_cgroup_zoneinfo(pc);
850         list_move(&pc->lru, &mz->lists[lru]);
851 }
852
853 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
854 {
855         struct page_cgroup *pc;
856         struct mem_cgroup_per_zone *mz;
857
858         if (mem_cgroup_disabled())
859                 return;
860         pc = lookup_page_cgroup(page);
861         VM_BUG_ON(PageCgroupAcctLRU(pc));
862         /*
863          * Used bit is set without atomic ops but after smp_wmb().
864          * For making pc->mem_cgroup visible, insert smp_rmb() here.
865          */
866         smp_rmb();
867         if (!PageCgroupUsed(pc))
868                 return;
869
870         mz = page_cgroup_zoneinfo(pc);
871         MEM_CGROUP_ZSTAT(mz, lru) += 1;
872         SetPageCgroupAcctLRU(pc);
873         if (mem_cgroup_is_root(pc->mem_cgroup))
874                 return;
875         list_add(&pc->lru, &mz->lists[lru]);
876 }
877
878 /*
879  * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
880  * lru because the page may.be reused after it's fully uncharged (because of
881  * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
882  * it again. This function is only used to charge SwapCache. It's done under
883  * lock_page and expected that zone->lru_lock is never held.
884  */
885 static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
886 {
887         unsigned long flags;
888         struct zone *zone = page_zone(page);
889         struct page_cgroup *pc = lookup_page_cgroup(page);
890
891         spin_lock_irqsave(&zone->lru_lock, flags);
892         /*
893          * Forget old LRU when this page_cgroup is *not* used. This Used bit
894          * is guarded by lock_page() because the page is SwapCache.
895          */
896         if (!PageCgroupUsed(pc))
897                 mem_cgroup_del_lru_list(page, page_lru(page));
898         spin_unlock_irqrestore(&zone->lru_lock, flags);
899 }
900
901 static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
902 {
903         unsigned long flags;
904         struct zone *zone = page_zone(page);
905         struct page_cgroup *pc = lookup_page_cgroup(page);
906
907         spin_lock_irqsave(&zone->lru_lock, flags);
908         /* link when the page is linked to LRU but page_cgroup isn't */
909         if (PageLRU(page) && !PageCgroupAcctLRU(pc))
910                 mem_cgroup_add_lru_list(page, page_lru(page));
911         spin_unlock_irqrestore(&zone->lru_lock, flags);
912 }
913
914
915 void mem_cgroup_move_lists(struct page *page,
916                            enum lru_list from, enum lru_list to)
917 {
918         if (mem_cgroup_disabled())
919                 return;
920         mem_cgroup_del_lru_list(page, from);
921         mem_cgroup_add_lru_list(page, to);
922 }
923
924 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
925 {
926         int ret;
927         struct mem_cgroup *curr = NULL;
928         struct task_struct *p;
929
930         p = find_lock_task_mm(task);
931         if (!p)
932                 return 0;
933         curr = try_get_mem_cgroup_from_mm(p->mm);
934         task_unlock(p);
935         if (!curr)
936                 return 0;
937         /*
938          * We should check use_hierarchy of "mem" not "curr". Because checking
939          * use_hierarchy of "curr" here make this function true if hierarchy is
940          * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
941          * hierarchy(even if use_hierarchy is disabled in "mem").
942          */
943         if (mem->use_hierarchy)
944                 ret = css_is_ancestor(&curr->css, &mem->css);
945         else
946                 ret = (curr == mem);
947         css_put(&curr->css);
948         return ret;
949 }
950
951 static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
952 {
953         unsigned long active;
954         unsigned long inactive;
955         unsigned long gb;
956         unsigned long inactive_ratio;
957
958         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
959         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
960
961         gb = (inactive + active) >> (30 - PAGE_SHIFT);
962         if (gb)
963                 inactive_ratio = int_sqrt(10 * gb);
964         else
965                 inactive_ratio = 1;
966
967         if (present_pages) {
968                 present_pages[0] = inactive;
969                 present_pages[1] = active;
970         }
971
972         return inactive_ratio;
973 }
974
975 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
976 {
977         unsigned long active;
978         unsigned long inactive;
979         unsigned long present_pages[2];
980         unsigned long inactive_ratio;
981
982         inactive_ratio = calc_inactive_ratio(memcg, present_pages);
983
984         inactive = present_pages[0];
985         active = present_pages[1];
986
987         if (inactive * inactive_ratio < active)
988                 return 1;
989
990         return 0;
991 }
992
993 int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
994 {
995         unsigned long active;
996         unsigned long inactive;
997
998         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
999         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
1000
1001         return (active > inactive);
1002 }
1003
1004 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
1005                                        struct zone *zone,
1006                                        enum lru_list lru)
1007 {
1008         int nid = zone_to_nid(zone);
1009         int zid = zone_idx(zone);
1010         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1011
1012         return MEM_CGROUP_ZSTAT(mz, lru);
1013 }
1014
1015 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
1016                                                       struct zone *zone)
1017 {
1018         int nid = zone_to_nid(zone);
1019         int zid = zone_idx(zone);
1020         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1021
1022         return &mz->reclaim_stat;
1023 }
1024
1025 struct zone_reclaim_stat *
1026 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
1027 {
1028         struct page_cgroup *pc;
1029         struct mem_cgroup_per_zone *mz;
1030
1031         if (mem_cgroup_disabled())
1032                 return NULL;
1033
1034         pc = lookup_page_cgroup(page);
1035         /*
1036          * Used bit is set without atomic ops but after smp_wmb().
1037          * For making pc->mem_cgroup visible, insert smp_rmb() here.
1038          */
1039         smp_rmb();
1040         if (!PageCgroupUsed(pc))
1041                 return NULL;
1042
1043         mz = page_cgroup_zoneinfo(pc);
1044         if (!mz)
1045                 return NULL;
1046
1047         return &mz->reclaim_stat;
1048 }
1049
1050 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
1051                                         struct list_head *dst,
1052                                         unsigned long *scanned, int order,
1053                                         int mode, struct zone *z,
1054                                         struct mem_cgroup *mem_cont,
1055                                         int active, int file)
1056 {
1057         unsigned long nr_taken = 0;
1058         struct page *page;
1059         unsigned long scan;
1060         LIST_HEAD(pc_list);
1061         struct list_head *src;
1062         struct page_cgroup *pc, *tmp;
1063         int nid = zone_to_nid(z);
1064         int zid = zone_idx(z);
1065         struct mem_cgroup_per_zone *mz;
1066         int lru = LRU_FILE * file + active;
1067         int ret;
1068
1069         BUG_ON(!mem_cont);
1070         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
1071         src = &mz->lists[lru];
1072
1073         scan = 0;
1074         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
1075                 if (scan >= nr_to_scan)
1076                         break;
1077
1078                 page = pc->page;
1079                 if (unlikely(!PageCgroupUsed(pc)))
1080                         continue;
1081                 if (unlikely(!PageLRU(page)))
1082                         continue;
1083
1084                 scan++;
1085                 ret = __isolate_lru_page(page, mode, file);
1086                 switch (ret) {
1087                 case 0:
1088                         list_move(&page->lru, dst);
1089                         mem_cgroup_del_lru(page);
1090                         nr_taken++;
1091                         break;
1092                 case -EBUSY:
1093                         /* we don't affect global LRU but rotate in our LRU */
1094                         mem_cgroup_rotate_lru_list(page, page_lru(page));
1095                         break;
1096                 default:
1097                         break;
1098                 }
1099         }
1100
1101         *scanned = scan;
1102
1103         trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1104                                       0, 0, 0, mode);
1105
1106         return nr_taken;
1107 }
1108
1109 #define mem_cgroup_from_res_counter(counter, member)    \
1110         container_of(counter, struct mem_cgroup, member)
1111
1112 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1113 {
1114         if (do_swap_account) {
1115                 if (res_counter_check_under_limit(&mem->res) &&
1116                         res_counter_check_under_limit(&mem->memsw))
1117                         return true;
1118         } else
1119                 if (res_counter_check_under_limit(&mem->res))
1120                         return true;
1121         return false;
1122 }
1123
1124 static unsigned int get_swappiness(struct mem_cgroup *memcg)
1125 {
1126         struct cgroup *cgrp = memcg->css.cgroup;
1127         unsigned int swappiness;
1128
1129         /* root ? */
1130         if (cgrp->parent == NULL)
1131                 return vm_swappiness;
1132
1133         spin_lock(&memcg->reclaim_param_lock);
1134         swappiness = memcg->swappiness;
1135         spin_unlock(&memcg->reclaim_param_lock);
1136
1137         return swappiness;
1138 }
1139
1140 static void mem_cgroup_start_move(struct mem_cgroup *mem)
1141 {
1142         int cpu;
1143
1144         get_online_cpus();
1145         spin_lock(&mem->pcp_counter_lock);
1146         for_each_online_cpu(cpu)
1147                 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
1148         mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
1149         spin_unlock(&mem->pcp_counter_lock);
1150         put_online_cpus();
1151
1152         synchronize_rcu();
1153 }
1154
1155 static void mem_cgroup_end_move(struct mem_cgroup *mem)
1156 {
1157         int cpu;
1158
1159         if (!mem)
1160                 return;
1161         get_online_cpus();
1162         spin_lock(&mem->pcp_counter_lock);
1163         for_each_online_cpu(cpu)
1164                 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
1165         mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
1166         spin_unlock(&mem->pcp_counter_lock);
1167         put_online_cpus();
1168 }
1169 /*
1170  * 2 routines for checking "mem" is under move_account() or not.
1171  *
1172  * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
1173  *                        for avoiding race in accounting. If true,
1174  *                        pc->mem_cgroup may be overwritten.
1175  *
1176  * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1177  *                        under hierarchy of moving cgroups. This is for
1178  *                        waiting at hith-memory prressure caused by "move".
1179  */
1180
1181 static bool mem_cgroup_stealed(struct mem_cgroup *mem)
1182 {
1183         VM_BUG_ON(!rcu_read_lock_held());
1184         return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1185 }
1186
1187 static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1188 {
1189         struct mem_cgroup *from;
1190         struct mem_cgroup *to;
1191         bool ret = false;
1192         /*
1193          * Unlike task_move routines, we access mc.to, mc.from not under
1194          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1195          */
1196         spin_lock(&mc.lock);
1197         from = mc.from;
1198         to = mc.to;
1199         if (!from)
1200                 goto unlock;
1201         if (from == mem || to == mem
1202             || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
1203             || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
1204                 ret = true;
1205 unlock:
1206         spin_unlock(&mc.lock);
1207         return ret;
1208 }
1209
1210 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1211 {
1212         if (mc.moving_task && current != mc.moving_task) {
1213                 if (mem_cgroup_under_move(mem)) {
1214                         DEFINE_WAIT(wait);
1215                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1216                         /* moving charge context might have finished. */
1217                         if (mc.moving_task)
1218                                 schedule();
1219                         finish_wait(&mc.waitq, &wait);
1220                         return true;
1221                 }
1222         }
1223         return false;
1224 }
1225
1226 /**
1227  * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1228  * @memcg: The memory cgroup that went over limit
1229  * @p: Task that is going to be killed
1230  *
1231  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1232  * enabled
1233  */
1234 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1235 {
1236         struct cgroup *task_cgrp;
1237         struct cgroup *mem_cgrp;
1238         /*
1239          * Need a buffer in BSS, can't rely on allocations. The code relies
1240          * on the assumption that OOM is serialized for memory controller.
1241          * If this assumption is broken, revisit this code.
1242          */
1243         static char memcg_name[PATH_MAX];
1244         int ret;
1245
1246         if (!memcg || !p)
1247                 return;
1248
1249
1250         rcu_read_lock();
1251
1252         mem_cgrp = memcg->css.cgroup;
1253         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1254
1255         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1256         if (ret < 0) {
1257                 /*
1258                  * Unfortunately, we are unable to convert to a useful name
1259                  * But we'll still print out the usage information
1260                  */
1261                 rcu_read_unlock();
1262                 goto done;
1263         }
1264         rcu_read_unlock();
1265
1266         printk(KERN_INFO "Task in %s killed", memcg_name);
1267
1268         rcu_read_lock();
1269         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1270         if (ret < 0) {
1271                 rcu_read_unlock();
1272                 goto done;
1273         }
1274         rcu_read_unlock();
1275
1276         /*
1277          * Continues from above, so we don't need an KERN_ level
1278          */
1279         printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1280 done:
1281
1282         printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1283                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1284                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1285                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1286         printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1287                 "failcnt %llu\n",
1288                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1289                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1290                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1291 }
1292
1293 /*
1294  * This function returns the number of memcg under hierarchy tree. Returns
1295  * 1(self count) if no children.
1296  */
1297 static int mem_cgroup_count_children(struct mem_cgroup *mem)
1298 {
1299         int num = 0;
1300         struct mem_cgroup *iter;
1301
1302         for_each_mem_cgroup_tree(iter, mem)
1303                 num++;
1304         return num;
1305 }
1306
1307 /*
1308  * Return the memory (and swap, if configured) limit for a memcg.
1309  */
1310 u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1311 {
1312         u64 limit;
1313         u64 memsw;
1314
1315         limit = res_counter_read_u64(&memcg->res, RES_LIMIT) +
1316                         total_swap_pages;
1317         memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1318         /*
1319          * If memsw is finite and limits the amount of swap space available
1320          * to this memcg, return that limit.
1321          */
1322         return min(limit, memsw);
1323 }
1324
1325 /*
1326  * Visit the first child (need not be the first child as per the ordering
1327  * of the cgroup list, since we track last_scanned_child) of @mem and use
1328  * that to reclaim free pages from.
1329  */
1330 static struct mem_cgroup *
1331 mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1332 {
1333         struct mem_cgroup *ret = NULL;
1334         struct cgroup_subsys_state *css;
1335         int nextid, found;
1336
1337         if (!root_mem->use_hierarchy) {
1338                 css_get(&root_mem->css);
1339                 ret = root_mem;
1340         }
1341
1342         while (!ret) {
1343                 rcu_read_lock();
1344                 nextid = root_mem->last_scanned_child + 1;
1345                 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1346                                    &found);
1347                 if (css && css_tryget(css))
1348                         ret = container_of(css, struct mem_cgroup, css);
1349
1350                 rcu_read_unlock();
1351                 /* Updates scanning parameter */
1352                 spin_lock(&root_mem->reclaim_param_lock);
1353                 if (!css) {
1354                         /* this means start scan from ID:1 */
1355                         root_mem->last_scanned_child = 0;
1356                 } else
1357                         root_mem->last_scanned_child = found;
1358                 spin_unlock(&root_mem->reclaim_param_lock);
1359         }
1360
1361         return ret;
1362 }
1363
1364 /*
1365  * Scan the hierarchy if needed to reclaim memory. We remember the last child
1366  * we reclaimed from, so that we don't end up penalizing one child extensively
1367  * based on its position in the children list.
1368  *
1369  * root_mem is the original ancestor that we've been reclaim from.
1370  *
1371  * We give up and return to the caller when we visit root_mem twice.
1372  * (other groups can be removed while we're walking....)
1373  *
1374  * If shrink==true, for avoiding to free too much, this returns immedieately.
1375  */
1376 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
1377                                                 struct zone *zone,
1378                                                 gfp_t gfp_mask,
1379                                                 unsigned long reclaim_options)
1380 {
1381         struct mem_cgroup *victim;
1382         int ret, total = 0;
1383         int loop = 0;
1384         bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1385         bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1386         bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1387         unsigned long excess = mem_cgroup_get_excess(root_mem);
1388
1389         /* If memsw_is_minimum==1, swap-out is of-no-use. */
1390         if (root_mem->memsw_is_minimum)
1391                 noswap = true;
1392
1393         while (1) {
1394                 victim = mem_cgroup_select_victim(root_mem);
1395                 if (victim == root_mem) {
1396                         loop++;
1397                         if (loop >= 1)
1398                                 drain_all_stock_async();
1399                         if (loop >= 2) {
1400                                 /*
1401                                  * If we have not been able to reclaim
1402                                  * anything, it might because there are
1403                                  * no reclaimable pages under this hierarchy
1404                                  */
1405                                 if (!check_soft || !total) {
1406                                         css_put(&victim->css);
1407                                         break;
1408                                 }
1409                                 /*
1410                                  * We want to do more targetted reclaim.
1411                                  * excess >> 2 is not to excessive so as to
1412                                  * reclaim too much, nor too less that we keep
1413                                  * coming back to reclaim from this cgroup
1414                                  */
1415                                 if (total >= (excess >> 2) ||
1416                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1417                                         css_put(&victim->css);
1418                                         break;
1419                                 }
1420                         }
1421                 }
1422                 if (!mem_cgroup_local_usage(victim)) {
1423                         /* this cgroup's local usage == 0 */
1424                         css_put(&victim->css);
1425                         continue;
1426                 }
1427                 /* we use swappiness of local cgroup */
1428                 if (check_soft)
1429                         ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
1430                                 noswap, get_swappiness(victim), zone);
1431                 else
1432                         ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1433                                                 noswap, get_swappiness(victim));
1434                 css_put(&victim->css);
1435                 /*
1436                  * At shrinking usage, we can't check we should stop here or
1437                  * reclaim more. It's depends on callers. last_scanned_child
1438                  * will work enough for keeping fairness under tree.
1439                  */
1440                 if (shrink)
1441                         return ret;
1442                 total += ret;
1443                 if (check_soft) {
1444                         if (res_counter_check_under_soft_limit(&root_mem->res))
1445                                 return total;
1446                 } else if (mem_cgroup_check_under_limit(root_mem))
1447                         return 1 + total;
1448         }
1449         return total;
1450 }
1451
1452 /*
1453  * Check OOM-Killer is already running under our hierarchy.
1454  * If someone is running, return false.
1455  */
1456 static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1457 {
1458         int x, lock_count = 0;
1459         struct mem_cgroup *iter;
1460
1461         for_each_mem_cgroup_tree(iter, mem) {
1462                 x = atomic_inc_return(&iter->oom_lock);
1463                 lock_count = max(x, lock_count);
1464         }
1465
1466         if (lock_count == 1)
1467                 return true;
1468         return false;
1469 }
1470
1471 static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
1472 {
1473         struct mem_cgroup *iter;
1474
1475         /*
1476          * When a new child is created while the hierarchy is under oom,
1477          * mem_cgroup_oom_lock() may not be called. We have to use
1478          * atomic_add_unless() here.
1479          */
1480         for_each_mem_cgroup_tree(iter, mem)
1481                 atomic_add_unless(&iter->oom_lock, -1, 0);
1482         return 0;
1483 }
1484
1485
1486 static DEFINE_MUTEX(memcg_oom_mutex);
1487 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1488
1489 struct oom_wait_info {
1490         struct mem_cgroup *mem;
1491         wait_queue_t    wait;
1492 };
1493
1494 static int memcg_oom_wake_function(wait_queue_t *wait,
1495         unsigned mode, int sync, void *arg)
1496 {
1497         struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1498         struct oom_wait_info *oom_wait_info;
1499
1500         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1501
1502         if (oom_wait_info->mem == wake_mem)
1503                 goto wakeup;
1504         /* if no hierarchy, no match */
1505         if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1506                 return 0;
1507         /*
1508          * Both of oom_wait_info->mem and wake_mem are stable under us.
1509          * Then we can use css_is_ancestor without taking care of RCU.
1510          */
1511         if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1512             !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1513                 return 0;
1514
1515 wakeup:
1516         return autoremove_wake_function(wait, mode, sync, arg);
1517 }
1518
1519 static void memcg_wakeup_oom(struct mem_cgroup *mem)
1520 {
1521         /* for filtering, pass "mem" as argument. */
1522         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1523 }
1524
1525 static void memcg_oom_recover(struct mem_cgroup *mem)
1526 {
1527         if (mem && atomic_read(&mem->oom_lock))
1528                 memcg_wakeup_oom(mem);
1529 }
1530
1531 /*
1532  * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1533  */
1534 bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1535 {
1536         struct oom_wait_info owait;
1537         bool locked, need_to_kill;
1538
1539         owait.mem = mem;
1540         owait.wait.flags = 0;
1541         owait.wait.func = memcg_oom_wake_function;
1542         owait.wait.private = current;
1543         INIT_LIST_HEAD(&owait.wait.task_list);
1544         need_to_kill = true;
1545         /* At first, try to OOM lock hierarchy under mem.*/
1546         mutex_lock(&memcg_oom_mutex);
1547         locked = mem_cgroup_oom_lock(mem);
1548         /*
1549          * Even if signal_pending(), we can't quit charge() loop without
1550          * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1551          * under OOM is always welcomed, use TASK_KILLABLE here.
1552          */
1553         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1554         if (!locked || mem->oom_kill_disable)
1555                 need_to_kill = false;
1556         if (locked)
1557                 mem_cgroup_oom_notify(mem);
1558         mutex_unlock(&memcg_oom_mutex);
1559
1560         if (need_to_kill) {
1561                 finish_wait(&memcg_oom_waitq, &owait.wait);
1562                 mem_cgroup_out_of_memory(mem, mask);
1563         } else {
1564                 schedule();
1565                 finish_wait(&memcg_oom_waitq, &owait.wait);
1566         }
1567         mutex_lock(&memcg_oom_mutex);
1568         mem_cgroup_oom_unlock(mem);
1569         memcg_wakeup_oom(mem);
1570         mutex_unlock(&memcg_oom_mutex);
1571
1572         if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1573                 return false;
1574         /* Give chance to dying process */
1575         schedule_timeout(1);
1576         return true;
1577 }
1578
1579 /*
1580  * Currently used to update mapped file statistics, but the routine can be
1581  * generalized to update other statistics as well.
1582  *
1583  * Notes: Race condition
1584  *
1585  * We usually use page_cgroup_lock() for accessing page_cgroup member but
1586  * it tends to be costly. But considering some conditions, we doesn't need
1587  * to do so _always_.
1588  *
1589  * Considering "charge", lock_page_cgroup() is not required because all
1590  * file-stat operations happen after a page is attached to radix-tree. There
1591  * are no race with "charge".
1592  *
1593  * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
1594  * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
1595  * if there are race with "uncharge". Statistics itself is properly handled
1596  * by flags.
1597  *
1598  * Considering "move", this is an only case we see a race. To make the race
1599  * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
1600  * possibility of race condition. If there is, we take a lock.
1601  */
1602
1603 static void mem_cgroup_update_file_stat(struct page *page, int idx, int val)
1604 {
1605         struct mem_cgroup *mem;
1606         struct page_cgroup *pc = lookup_page_cgroup(page);
1607         bool need_unlock = false;
1608
1609         if (unlikely(!pc))
1610                 return;
1611
1612         rcu_read_lock();
1613         mem = pc->mem_cgroup;
1614         if (unlikely(!mem || !PageCgroupUsed(pc)))
1615                 goto out;
1616         /* pc->mem_cgroup is unstable ? */
1617         if (unlikely(mem_cgroup_stealed(mem))) {
1618                 /* take a lock against to access pc->mem_cgroup */
1619                 lock_page_cgroup(pc);
1620                 need_unlock = true;
1621                 mem = pc->mem_cgroup;
1622                 if (!mem || !PageCgroupUsed(pc))
1623                         goto out;
1624         }
1625
1626         this_cpu_add(mem->stat->count[idx], val);
1627
1628         switch (idx) {
1629         case MEM_CGROUP_STAT_FILE_MAPPED:
1630                 if (val > 0)
1631                         SetPageCgroupFileMapped(pc);
1632                 else if (!page_mapped(page))
1633                         ClearPageCgroupFileMapped(pc);
1634                 break;
1635         default:
1636                 BUG();
1637         }
1638
1639 out:
1640         if (unlikely(need_unlock))
1641                 unlock_page_cgroup(pc);
1642         rcu_read_unlock();
1643         return;
1644 }
1645
1646 void mem_cgroup_update_file_mapped(struct page *page, int val)
1647 {
1648         mem_cgroup_update_file_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, val);
1649 }
1650
1651 /*
1652  * size of first charge trial. "32" comes from vmscan.c's magic value.
1653  * TODO: maybe necessary to use big numbers in big irons.
1654  */
1655 #define CHARGE_SIZE     (32 * PAGE_SIZE)
1656 struct memcg_stock_pcp {
1657         struct mem_cgroup *cached; /* this never be root cgroup */
1658         int charge;
1659         struct work_struct work;
1660 };
1661 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1662 static atomic_t memcg_drain_count;
1663
1664 /*
1665  * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1666  * from local stock and true is returned. If the stock is 0 or charges from a
1667  * cgroup which is not current target, returns false. This stock will be
1668  * refilled.
1669  */
1670 static bool consume_stock(struct mem_cgroup *mem)
1671 {
1672         struct memcg_stock_pcp *stock;
1673         bool ret = true;
1674
1675         stock = &get_cpu_var(memcg_stock);
1676         if (mem == stock->cached && stock->charge)
1677                 stock->charge -= PAGE_SIZE;
1678         else /* need to call res_counter_charge */
1679                 ret = false;
1680         put_cpu_var(memcg_stock);
1681         return ret;
1682 }
1683
1684 /*
1685  * Returns stocks cached in percpu to res_counter and reset cached information.
1686  */
1687 static void drain_stock(struct memcg_stock_pcp *stock)
1688 {
1689         struct mem_cgroup *old = stock->cached;
1690
1691         if (stock->charge) {
1692                 res_counter_uncharge(&old->res, stock->charge);
1693                 if (do_swap_account)
1694                         res_counter_uncharge(&old->memsw, stock->charge);
1695         }
1696         stock->cached = NULL;
1697         stock->charge = 0;
1698 }
1699
1700 /*
1701  * This must be called under preempt disabled or must be called by
1702  * a thread which is pinned to local cpu.
1703  */
1704 static void drain_local_stock(struct work_struct *dummy)
1705 {
1706         struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1707         drain_stock(stock);
1708 }
1709
1710 /*
1711  * Cache charges(val) which is from res_counter, to local per_cpu area.
1712  * This will be consumed by consume_stock() function, later.
1713  */
1714 static void refill_stock(struct mem_cgroup *mem, int val)
1715 {
1716         struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1717
1718         if (stock->cached != mem) { /* reset if necessary */
1719                 drain_stock(stock);
1720                 stock->cached = mem;
1721         }
1722         stock->charge += val;
1723         put_cpu_var(memcg_stock);
1724 }
1725
1726 /*
1727  * Tries to drain stocked charges in other cpus. This function is asynchronous
1728  * and just put a work per cpu for draining localy on each cpu. Caller can
1729  * expects some charges will be back to res_counter later but cannot wait for
1730  * it.
1731  */
1732 static void drain_all_stock_async(void)
1733 {
1734         int cpu;
1735         /* This function is for scheduling "drain" in asynchronous way.
1736          * The result of "drain" is not directly handled by callers. Then,
1737          * if someone is calling drain, we don't have to call drain more.
1738          * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1739          * there is a race. We just do loose check here.
1740          */
1741         if (atomic_read(&memcg_drain_count))
1742                 return;
1743         /* Notify other cpus that system-wide "drain" is running */
1744         atomic_inc(&memcg_drain_count);
1745         get_online_cpus();
1746         for_each_online_cpu(cpu) {
1747                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1748                 schedule_work_on(cpu, &stock->work);
1749         }
1750         put_online_cpus();
1751         atomic_dec(&memcg_drain_count);
1752         /* We don't wait for flush_work */
1753 }
1754
1755 /* This is a synchronous drain interface. */
1756 static void drain_all_stock_sync(void)
1757 {
1758         /* called when force_empty is called */
1759         atomic_inc(&memcg_drain_count);
1760         schedule_on_each_cpu(drain_local_stock);
1761         atomic_dec(&memcg_drain_count);
1762 }
1763
1764 /*
1765  * This function drains percpu counter value from DEAD cpu and
1766  * move it to local cpu. Note that this function can be preempted.
1767  */
1768 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
1769 {
1770         int i;
1771
1772         spin_lock(&mem->pcp_counter_lock);
1773         for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
1774                 s64 x = per_cpu(mem->stat->count[i], cpu);
1775
1776                 per_cpu(mem->stat->count[i], cpu) = 0;
1777                 mem->nocpu_base.count[i] += x;
1778         }
1779         /* need to clear ON_MOVE value, works as a kind of lock. */
1780         per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
1781         spin_unlock(&mem->pcp_counter_lock);
1782 }
1783
1784 static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
1785 {
1786         int idx = MEM_CGROUP_ON_MOVE;
1787
1788         spin_lock(&mem->pcp_counter_lock);
1789         per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
1790         spin_unlock(&mem->pcp_counter_lock);
1791 }
1792
1793 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
1794                                         unsigned long action,
1795                                         void *hcpu)
1796 {
1797         int cpu = (unsigned long)hcpu;
1798         struct memcg_stock_pcp *stock;
1799         struct mem_cgroup *iter;
1800
1801         if ((action == CPU_ONLINE)) {
1802                 for_each_mem_cgroup_all(iter)
1803                         synchronize_mem_cgroup_on_move(iter, cpu);
1804                 return NOTIFY_OK;
1805         }
1806
1807         if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
1808                 return NOTIFY_OK;
1809
1810         for_each_mem_cgroup_all(iter)
1811                 mem_cgroup_drain_pcp_counter(iter, cpu);
1812
1813         stock = &per_cpu(memcg_stock, cpu);
1814         drain_stock(stock);
1815         return NOTIFY_OK;
1816 }
1817
1818
1819 /* See __mem_cgroup_try_charge() for details */
1820 enum {
1821         CHARGE_OK,              /* success */
1822         CHARGE_RETRY,           /* need to retry but retry is not bad */
1823         CHARGE_NOMEM,           /* we can't do more. return -ENOMEM */
1824         CHARGE_WOULDBLOCK,      /* GFP_WAIT wasn't set and no enough res. */
1825         CHARGE_OOM_DIE,         /* the current is killed because of OOM */
1826 };
1827
1828 static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1829                                 int csize, bool oom_check)
1830 {
1831         struct mem_cgroup *mem_over_limit;
1832         struct res_counter *fail_res;
1833         unsigned long flags = 0;
1834         int ret;
1835
1836         ret = res_counter_charge(&mem->res, csize, &fail_res);
1837
1838         if (likely(!ret)) {
1839                 if (!do_swap_account)
1840                         return CHARGE_OK;
1841                 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1842                 if (likely(!ret))
1843                         return CHARGE_OK;
1844
1845                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1846                 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1847         } else
1848                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1849
1850         if (csize > PAGE_SIZE) /* change csize and retry */
1851                 return CHARGE_RETRY;
1852
1853         if (!(gfp_mask & __GFP_WAIT))
1854                 return CHARGE_WOULDBLOCK;
1855
1856         ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1857                                         gfp_mask, flags);
1858         /*
1859          * try_to_free_mem_cgroup_pages() might not give us a full
1860          * picture of reclaim. Some pages are reclaimed and might be
1861          * moved to swap cache or just unmapped from the cgroup.
1862          * Check the limit again to see if the reclaim reduced the
1863          * current usage of the cgroup before giving up
1864          */
1865         if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1866                 return CHARGE_RETRY;
1867
1868         /*
1869          * At task move, charge accounts can be doubly counted. So, it's
1870          * better to wait until the end of task_move if something is going on.
1871          */
1872         if (mem_cgroup_wait_acct_move(mem_over_limit))
1873                 return CHARGE_RETRY;
1874
1875         /* If we don't need to call oom-killer at el, return immediately */
1876         if (!oom_check)
1877                 return CHARGE_NOMEM;
1878         /* check OOM */
1879         if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1880                 return CHARGE_OOM_DIE;
1881
1882         return CHARGE_RETRY;
1883 }
1884
1885 /*
1886  * Unlike exported interface, "oom" parameter is added. if oom==true,
1887  * oom-killer can be invoked.
1888  */
1889 static int __mem_cgroup_try_charge(struct mm_struct *mm,
1890                 gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom)
1891 {
1892         int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1893         struct mem_cgroup *mem = NULL;
1894         int ret;
1895         int csize = CHARGE_SIZE;
1896
1897         /*
1898          * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1899          * in system level. So, allow to go ahead dying process in addition to
1900          * MEMDIE process.
1901          */
1902         if (unlikely(test_thread_flag(TIF_MEMDIE)
1903                      || fatal_signal_pending(current)))
1904                 goto bypass;
1905
1906         /*
1907          * We always charge the cgroup the mm_struct belongs to.
1908          * The mm_struct's mem_cgroup changes on task migration if the
1909          * thread group leader migrates. It's possible that mm is not
1910          * set, if so charge the init_mm (happens for pagecache usage).
1911          */
1912         if (!*memcg && !mm)
1913                 goto bypass;
1914 again:
1915         if (*memcg) { /* css should be a valid one */
1916                 mem = *memcg;
1917                 VM_BUG_ON(css_is_removed(&mem->css));
1918                 if (mem_cgroup_is_root(mem))
1919                         goto done;
1920                 if (consume_stock(mem))
1921                         goto done;
1922                 css_get(&mem->css);
1923         } else {
1924                 struct task_struct *p;
1925
1926                 rcu_read_lock();
1927                 p = rcu_dereference(mm->owner);
1928                 /*
1929                  * Because we don't have task_lock(), "p" can exit.
1930                  * In that case, "mem" can point to root or p can be NULL with
1931                  * race with swapoff. Then, we have small risk of mis-accouning.
1932                  * But such kind of mis-account by race always happens because
1933                  * we don't have cgroup_mutex(). It's overkill and we allo that
1934                  * small race, here.
1935                  * (*) swapoff at el will charge against mm-struct not against
1936                  * task-struct. So, mm->owner can be NULL.
1937                  */
1938                 mem = mem_cgroup_from_task(p);
1939                 if (!mem || mem_cgroup_is_root(mem)) {
1940                         rcu_read_unlock();
1941                         goto done;
1942                 }
1943                 if (consume_stock(mem)) {
1944                         /*
1945                          * It seems dagerous to access memcg without css_get().
1946                          * But considering how consume_stok works, it's not
1947                          * necessary. If consume_stock success, some charges
1948                          * from this memcg are cached on this cpu. So, we
1949                          * don't need to call css_get()/css_tryget() before
1950                          * calling consume_stock().
1951                          */
1952                         rcu_read_unlock();
1953                         goto done;
1954                 }
1955                 /* after here, we may be blocked. we need to get refcnt */
1956                 if (!css_tryget(&mem->css)) {
1957                         rcu_read_unlock();
1958                         goto again;
1959                 }
1960                 rcu_read_unlock();
1961         }
1962
1963         do {
1964                 bool oom_check;
1965
1966                 /* If killed, bypass charge */
1967                 if (fatal_signal_pending(current)) {
1968                         css_put(&mem->css);
1969                         goto bypass;
1970                 }
1971
1972                 oom_check = false;
1973                 if (oom && !nr_oom_retries) {
1974                         oom_check = true;
1975                         nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1976                 }
1977
1978                 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1979
1980                 switch (ret) {
1981                 case CHARGE_OK:
1982                         break;
1983                 case CHARGE_RETRY: /* not in OOM situation but retry */
1984                         csize = PAGE_SIZE;
1985                         css_put(&mem->css);
1986                         mem = NULL;
1987                         goto again;
1988                 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
1989                         css_put(&mem->css);
1990                         goto nomem;
1991                 case CHARGE_NOMEM: /* OOM routine works */
1992                         if (!oom) {
1993                                 css_put(&mem->css);
1994                                 goto nomem;
1995                         }
1996                         /* If oom, we never return -ENOMEM */
1997                         nr_oom_retries--;
1998                         break;
1999                 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2000                         css_put(&mem->css);
2001                         goto bypass;
2002                 }
2003         } while (ret != CHARGE_OK);
2004
2005         if (csize > PAGE_SIZE)
2006                 refill_stock(mem, csize - PAGE_SIZE);
2007         css_put(&mem->css);
2008 done:
2009         *memcg = mem;
2010         return 0;
2011 nomem:
2012         *memcg = NULL;
2013         return -ENOMEM;
2014 bypass:
2015         *memcg = NULL;
2016         return 0;
2017 }
2018
2019 /*
2020  * Somemtimes we have to undo a charge we got by try_charge().
2021  * This function is for that and do uncharge, put css's refcnt.
2022  * gotten by try_charge().
2023  */
2024 static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2025                                                         unsigned long count)
2026 {
2027         if (!mem_cgroup_is_root(mem)) {
2028                 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
2029                 if (do_swap_account)
2030                         res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
2031         }
2032 }
2033
2034 static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
2035 {
2036         __mem_cgroup_cancel_charge(mem, 1);
2037 }
2038
2039 /*
2040  * A helper function to get mem_cgroup from ID. must be called under
2041  * rcu_read_lock(). The caller must check css_is_removed() or some if
2042  * it's concern. (dropping refcnt from swap can be called against removed
2043  * memcg.)
2044  */
2045 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2046 {
2047         struct cgroup_subsys_state *css;
2048
2049         /* ID 0 is unused ID */
2050         if (!id)
2051                 return NULL;
2052         css = css_lookup(&mem_cgroup_subsys, id);
2053         if (!css)
2054                 return NULL;
2055         return container_of(css, struct mem_cgroup, css);
2056 }
2057
2058 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2059 {
2060         struct mem_cgroup *mem = NULL;
2061         struct page_cgroup *pc;
2062         unsigned short id;
2063         swp_entry_t ent;
2064
2065         VM_BUG_ON(!PageLocked(page));
2066
2067         pc = lookup_page_cgroup(page);
2068         lock_page_cgroup(pc);
2069         if (PageCgroupUsed(pc)) {
2070                 mem = pc->mem_cgroup;
2071                 if (mem && !css_tryget(&mem->css))
2072                         mem = NULL;
2073         } else if (PageSwapCache(page)) {
2074                 ent.val = page_private(page);
2075                 id = lookup_swap_cgroup(ent);
2076                 rcu_read_lock();
2077                 mem = mem_cgroup_lookup(id);
2078                 if (mem && !css_tryget(&mem->css))
2079                         mem = NULL;
2080                 rcu_read_unlock();
2081         }
2082         unlock_page_cgroup(pc);
2083         return mem;
2084 }
2085
2086 /*
2087  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
2088  * USED state. If already USED, uncharge and return.
2089  */
2090
2091 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
2092                                      struct page_cgroup *pc,
2093                                      enum charge_type ctype)
2094 {
2095         /* try_charge() can return NULL to *memcg, taking care of it. */
2096         if (!mem)
2097                 return;
2098
2099         lock_page_cgroup(pc);
2100         if (unlikely(PageCgroupUsed(pc))) {
2101                 unlock_page_cgroup(pc);
2102                 mem_cgroup_cancel_charge(mem);
2103                 return;
2104         }
2105
2106         pc->mem_cgroup = mem;
2107         /*
2108          * We access a page_cgroup asynchronously without lock_page_cgroup().
2109          * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2110          * is accessed after testing USED bit. To make pc->mem_cgroup visible
2111          * before USED bit, we need memory barrier here.
2112          * See mem_cgroup_add_lru_list(), etc.
2113          */
2114         smp_wmb();
2115         switch (ctype) {
2116         case MEM_CGROUP_CHARGE_TYPE_CACHE:
2117         case MEM_CGROUP_CHARGE_TYPE_SHMEM:
2118                 SetPageCgroupCache(pc);
2119                 SetPageCgroupUsed(pc);
2120                 break;
2121         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2122                 ClearPageCgroupCache(pc);
2123                 SetPageCgroupUsed(pc);
2124                 break;
2125         default:
2126                 break;
2127         }
2128
2129         mem_cgroup_charge_statistics(mem, pc, true);
2130
2131         unlock_page_cgroup(pc);
2132         /*
2133          * "charge_statistics" updated event counter. Then, check it.
2134          * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2135          * if they exceeds softlimit.
2136          */
2137         memcg_check_events(mem, pc->page);
2138 }
2139
2140 /**
2141  * __mem_cgroup_move_account - move account of the page
2142  * @pc: page_cgroup of the page.
2143  * @from: mem_cgroup which the page is moved from.
2144  * @to: mem_cgroup which the page is moved to. @from != @to.
2145  * @uncharge: whether we should call uncharge and css_put against @from.
2146  *
2147  * The caller must confirm following.
2148  * - page is not on LRU (isolate_page() is useful.)
2149  * - the pc is locked, used, and ->mem_cgroup points to @from.
2150  *
2151  * This function doesn't do "charge" nor css_get to new cgroup. It should be
2152  * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
2153  * true, this function does "uncharge" from old cgroup, but it doesn't if
2154  * @uncharge is false, so a caller should do "uncharge".
2155  */
2156
2157 static void __mem_cgroup_move_account(struct page_cgroup *pc,
2158         struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
2159 {
2160         VM_BUG_ON(from == to);
2161         VM_BUG_ON(PageLRU(pc->page));
2162         VM_BUG_ON(!page_is_cgroup_locked(pc));
2163         VM_BUG_ON(!PageCgroupUsed(pc));
2164         VM_BUG_ON(pc->mem_cgroup != from);
2165
2166         if (PageCgroupFileMapped(pc)) {
2167                 /* Update mapped_file data for mem_cgroup */
2168                 preempt_disable();
2169                 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2170                 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2171                 preempt_enable();
2172         }
2173         mem_cgroup_charge_statistics(from, pc, false);
2174         if (uncharge)
2175                 /* This is not "cancel", but cancel_charge does all we need. */
2176                 mem_cgroup_cancel_charge(from);
2177
2178         /* caller should have done css_get */
2179         pc->mem_cgroup = to;
2180         mem_cgroup_charge_statistics(to, pc, true);
2181         /*
2182          * We charges against "to" which may not have any tasks. Then, "to"
2183          * can be under rmdir(). But in current implementation, caller of
2184          * this function is just force_empty() and move charge, so it's
2185          * garanteed that "to" is never removed. So, we don't check rmdir
2186          * status here.
2187          */
2188 }
2189
2190 /*
2191  * check whether the @pc is valid for moving account and call
2192  * __mem_cgroup_move_account()
2193  */
2194 static int mem_cgroup_move_account(struct page_cgroup *pc,
2195                 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
2196 {
2197         int ret = -EINVAL;
2198         lock_page_cgroup(pc);
2199         if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
2200                 __mem_cgroup_move_account(pc, from, to, uncharge);
2201                 ret = 0;
2202         }
2203         unlock_page_cgroup(pc);
2204         /*
2205          * check events
2206          */
2207         memcg_check_events(to, pc->page);
2208         memcg_check_events(from, pc->page);
2209         return ret;
2210 }
2211
2212 /*
2213  * move charges to its parent.
2214  */
2215
2216 static int mem_cgroup_move_parent(struct page_cgroup *pc,
2217                                   struct mem_cgroup *child,
2218                                   gfp_t gfp_mask)
2219 {
2220         struct page *page = pc->page;
2221         struct cgroup *cg = child->css.cgroup;
2222         struct cgroup *pcg = cg->parent;
2223         struct mem_cgroup *parent;
2224         int ret;
2225
2226         /* Is ROOT ? */
2227         if (!pcg)
2228                 return -EINVAL;
2229
2230         ret = -EBUSY;
2231         if (!get_page_unless_zero(page))
2232                 goto out;
2233         if (isolate_lru_page(page))
2234                 goto put;
2235
2236         parent = mem_cgroup_from_cont(pcg);
2237         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
2238         if (ret || !parent)
2239                 goto put_back;
2240
2241         ret = mem_cgroup_move_account(pc, child, parent, true);
2242         if (ret)
2243                 mem_cgroup_cancel_charge(parent);
2244 put_back:
2245         putback_lru_page(page);
2246 put:
2247         put_page(page);
2248 out:
2249         return ret;
2250 }
2251
2252 /*
2253  * Charge the memory controller for page usage.
2254  * Return
2255  * 0 if the charge was successful
2256  * < 0 if the cgroup is over its limit
2257  */
2258 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2259                                 gfp_t gfp_mask, enum charge_type ctype)
2260 {
2261         struct mem_cgroup *mem = NULL;
2262         struct page_cgroup *pc;
2263         int ret;
2264
2265         pc = lookup_page_cgroup(page);
2266         /* can happen at boot */
2267         if (unlikely(!pc))
2268                 return 0;
2269         prefetchw(pc);
2270
2271         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
2272         if (ret || !mem)
2273                 return ret;
2274
2275         __mem_cgroup_commit_charge(mem, pc, ctype);
2276         return 0;
2277 }
2278
2279 int mem_cgroup_newpage_charge(struct page *page,
2280                               struct mm_struct *mm, gfp_t gfp_mask)
2281 {
2282         if (mem_cgroup_disabled())
2283                 return 0;
2284         if (PageCompound(page))
2285                 return 0;
2286         /*
2287          * If already mapped, we don't have to account.
2288          * If page cache, page->mapping has address_space.
2289          * But page->mapping may have out-of-use anon_vma pointer,
2290          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2291          * is NULL.
2292          */
2293         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2294                 return 0;
2295         if (unlikely(!mm))
2296                 mm = &init_mm;
2297         return mem_cgroup_charge_common(page, mm, gfp_mask,
2298                                 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2299 }
2300
2301 static void
2302 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2303                                         enum charge_type ctype);
2304
2305 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2306                                 gfp_t gfp_mask)
2307 {
2308         int ret;
2309
2310         if (mem_cgroup_disabled())
2311                 return 0;
2312         if (PageCompound(page))
2313                 return 0;
2314         /*
2315          * Corner case handling. This is called from add_to_page_cache()
2316          * in usual. But some FS (shmem) precharges this page before calling it
2317          * and call add_to_page_cache() with GFP_NOWAIT.
2318          *
2319          * For GFP_NOWAIT case, the page may be pre-charged before calling
2320          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2321          * charge twice. (It works but has to pay a bit larger cost.)
2322          * And when the page is SwapCache, it should take swap information
2323          * into account. This is under lock_page() now.
2324          */
2325         if (!(gfp_mask & __GFP_WAIT)) {
2326                 struct page_cgroup *pc;
2327
2328                 pc = lookup_page_cgroup(page);
2329                 if (!pc)
2330                         return 0;
2331                 lock_page_cgroup(pc);
2332                 if (PageCgroupUsed(pc)) {
2333                         unlock_page_cgroup(pc);
2334                         return 0;
2335                 }
2336                 unlock_page_cgroup(pc);
2337         }
2338
2339         if (unlikely(!mm))
2340                 mm = &init_mm;
2341
2342         if (page_is_file_cache(page))
2343                 return mem_cgroup_charge_common(page, mm, gfp_mask,
2344                                 MEM_CGROUP_CHARGE_TYPE_CACHE);
2345
2346         /* shmem */
2347         if (PageSwapCache(page)) {
2348                 struct mem_cgroup *mem = NULL;
2349
2350                 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2351                 if (!ret)
2352                         __mem_cgroup_commit_charge_swapin(page, mem,
2353                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
2354         } else
2355                 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
2356                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
2357
2358         return ret;
2359 }
2360
2361 /*
2362  * While swap-in, try_charge -> commit or cancel, the page is locked.
2363  * And when try_charge() successfully returns, one refcnt to memcg without
2364  * struct page_cgroup is acquired. This refcnt will be consumed by
2365  * "commit()" or removed by "cancel()"
2366  */
2367 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2368                                  struct page *page,
2369                                  gfp_t mask, struct mem_cgroup **ptr)
2370 {
2371         struct mem_cgroup *mem;
2372         int ret;
2373
2374         if (mem_cgroup_disabled())
2375                 return 0;
2376
2377         if (!do_swap_account)
2378                 goto charge_cur_mm;
2379         /*
2380          * A racing thread's fault, or swapoff, may have already updated
2381          * the pte, and even removed page from swap cache: in those cases
2382          * do_swap_page()'s pte_same() test will fail; but there's also a
2383          * KSM case which does need to charge the page.
2384          */
2385         if (!PageSwapCache(page))
2386                 goto charge_cur_mm;
2387         mem = try_get_mem_cgroup_from_page(page);
2388         if (!mem)
2389                 goto charge_cur_mm;
2390         *ptr = mem;
2391         ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
2392         css_put(&mem->css);
2393         return ret;
2394 charge_cur_mm:
2395         if (unlikely(!mm))
2396                 mm = &init_mm;
2397         return __mem_cgroup_try_charge(mm, mask, ptr, true);
2398 }
2399
2400 static void
2401 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2402                                         enum charge_type ctype)
2403 {
2404         struct page_cgroup *pc;
2405
2406         if (mem_cgroup_disabled())
2407                 return;
2408         if (!ptr)
2409                 return;
2410         cgroup_exclude_rmdir(&ptr->css);
2411         pc = lookup_page_cgroup(page);
2412         mem_cgroup_lru_del_before_commit_swapcache(page);
2413         __mem_cgroup_commit_charge(ptr, pc, ctype);
2414         mem_cgroup_lru_add_after_commit_swapcache(page);
2415         /*
2416          * Now swap is on-memory. This means this page may be
2417          * counted both as mem and swap....double count.
2418          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2419          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2420          * may call delete_from_swap_cache() before reach here.
2421          */
2422         if (do_swap_account && PageSwapCache(page)) {
2423                 swp_entry_t ent = {.val = page_private(page)};
2424                 unsigned short id;
2425                 struct mem_cgroup *memcg;
2426
2427                 id = swap_cgroup_record(ent, 0);
2428                 rcu_read_lock();
2429                 memcg = mem_cgroup_lookup(id);
2430                 if (memcg) {
2431                         /*
2432                          * This recorded memcg can be obsolete one. So, avoid
2433                          * calling css_tryget
2434                          */
2435                         if (!mem_cgroup_is_root(memcg))
2436                                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2437                         mem_cgroup_swap_statistics(memcg, false);
2438                         mem_cgroup_put(memcg);
2439                 }
2440                 rcu_read_unlock();
2441         }
2442         /*
2443          * At swapin, we may charge account against cgroup which has no tasks.
2444          * So, rmdir()->pre_destroy() can be called while we do this charge.
2445          * In that case, we need to call pre_destroy() again. check it here.
2446          */
2447         cgroup_release_and_wakeup_rmdir(&ptr->css);
2448 }
2449
2450 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2451 {
2452         __mem_cgroup_commit_charge_swapin(page, ptr,
2453                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
2454 }
2455
2456 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2457 {
2458         if (mem_cgroup_disabled())
2459                 return;
2460         if (!mem)
2461                 return;
2462         mem_cgroup_cancel_charge(mem);
2463 }
2464
2465 static void
2466 __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
2467 {
2468         struct memcg_batch_info *batch = NULL;
2469         bool uncharge_memsw = true;
2470         /* If swapout, usage of swap doesn't decrease */
2471         if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2472                 uncharge_memsw = false;
2473
2474         batch = &current->memcg_batch;
2475         /*
2476          * In usual, we do css_get() when we remember memcg pointer.
2477          * But in this case, we keep res->usage until end of a series of
2478          * uncharges. Then, it's ok to ignore memcg's refcnt.
2479          */
2480         if (!batch->memcg)
2481                 batch->memcg = mem;
2482         /*
2483          * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2484          * In those cases, all pages freed continously can be expected to be in
2485          * the same cgroup and we have chance to coalesce uncharges.
2486          * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2487          * because we want to do uncharge as soon as possible.
2488          */
2489
2490         if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2491                 goto direct_uncharge;
2492
2493         /*
2494          * In typical case, batch->memcg == mem. This means we can
2495          * merge a series of uncharges to an uncharge of res_counter.
2496          * If not, we uncharge res_counter ony by one.
2497          */
2498         if (batch->memcg != mem)
2499                 goto direct_uncharge;
2500         /* remember freed charge and uncharge it later */
2501         batch->bytes += PAGE_SIZE;
2502         if (uncharge_memsw)
2503                 batch->memsw_bytes += PAGE_SIZE;
2504         return;
2505 direct_uncharge:
2506         res_counter_uncharge(&mem->res, PAGE_SIZE);
2507         if (uncharge_memsw)
2508                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
2509         if (unlikely(batch->memcg != mem))
2510                 memcg_oom_recover(mem);
2511         return;
2512 }
2513
2514 /*
2515  * uncharge if !page_mapped(page)
2516  */
2517 static struct mem_cgroup *
2518 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2519 {
2520         struct page_cgroup *pc;
2521         struct mem_cgroup *mem = NULL;
2522
2523         if (mem_cgroup_disabled())
2524                 return NULL;
2525
2526         if (PageSwapCache(page))
2527                 return NULL;
2528
2529         /*
2530          * Check if our page_cgroup is valid
2531          */
2532         pc = lookup_page_cgroup(page);
2533         if (unlikely(!pc || !PageCgroupUsed(pc)))
2534                 return NULL;
2535
2536         lock_page_cgroup(pc);
2537
2538         mem = pc->mem_cgroup;
2539
2540         if (!PageCgroupUsed(pc))
2541                 goto unlock_out;
2542
2543         switch (ctype) {
2544         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2545         case MEM_CGROUP_CHARGE_TYPE_DROP:
2546                 /* See mem_cgroup_prepare_migration() */
2547                 if (page_mapped(page) || PageCgroupMigration(pc))
2548                         goto unlock_out;
2549                 break;
2550         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2551                 if (!PageAnon(page)) {  /* Shared memory */
2552                         if (page->mapping && !page_is_file_cache(page))
2553                                 goto unlock_out;
2554                 } else if (page_mapped(page)) /* Anon */
2555                                 goto unlock_out;
2556                 break;
2557         default:
2558                 break;
2559         }
2560
2561         mem_cgroup_charge_statistics(mem, pc, false);
2562
2563         ClearPageCgroupUsed(pc);
2564         /*
2565          * pc->mem_cgroup is not cleared here. It will be accessed when it's
2566          * freed from LRU. This is safe because uncharged page is expected not
2567          * to be reused (freed soon). Exception is SwapCache, it's handled by
2568          * special functions.
2569          */
2570
2571         unlock_page_cgroup(pc);
2572         /*
2573          * even after unlock, we have mem->res.usage here and this memcg
2574          * will never be freed.
2575          */
2576         memcg_check_events(mem, page);
2577         if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
2578                 mem_cgroup_swap_statistics(mem, true);
2579                 mem_cgroup_get(mem);
2580         }
2581         if (!mem_cgroup_is_root(mem))
2582                 __do_uncharge(mem, ctype);
2583
2584         return mem;
2585
2586 unlock_out:
2587         unlock_page_cgroup(pc);
2588         return NULL;
2589 }
2590
2591 void mem_cgroup_uncharge_page(struct page *page)
2592 {
2593         /* early check. */
2594         if (page_mapped(page))
2595                 return;
2596         if (page->mapping && !PageAnon(page))
2597                 return;
2598         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2599 }
2600
2601 void mem_cgroup_uncharge_cache_page(struct page *page)
2602 {
2603         VM_BUG_ON(page_mapped(page));
2604         VM_BUG_ON(page->mapping);
2605         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2606 }
2607
2608 /*
2609  * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2610  * In that cases, pages are freed continuously and we can expect pages
2611  * are in the same memcg. All these calls itself limits the number of
2612  * pages freed at once, then uncharge_start/end() is called properly.
2613  * This may be called prural(2) times in a context,
2614  */
2615
2616 void mem_cgroup_uncharge_start(void)
2617 {
2618         current->memcg_batch.do_batch++;
2619         /* We can do nest. */
2620         if (current->memcg_batch.do_batch == 1) {
2621                 current->memcg_batch.memcg = NULL;
2622                 current->memcg_batch.bytes = 0;
2623                 current->memcg_batch.memsw_bytes = 0;
2624         }
2625 }
2626
2627 void mem_cgroup_uncharge_end(void)
2628 {
2629         struct memcg_batch_info *batch = &current->memcg_batch;
2630
2631         if (!batch->do_batch)
2632                 return;
2633
2634         batch->do_batch--;
2635         if (batch->do_batch) /* If stacked, do nothing. */
2636                 return;
2637
2638         if (!batch->memcg)
2639                 return;
2640         /*
2641          * This "batch->memcg" is valid without any css_get/put etc...
2642          * bacause we hide charges behind us.
2643          */
2644         if (batch->bytes)
2645                 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2646         if (batch->memsw_bytes)
2647                 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
2648         memcg_oom_recover(batch->memcg);
2649         /* forget this pointer (for sanity check) */
2650         batch->memcg = NULL;
2651 }
2652
2653 #ifdef CONFIG_SWAP
2654 /*
2655  * called after __delete_from_swap_cache() and drop "page" account.
2656  * memcg information is recorded to swap_cgroup of "ent"
2657  */
2658 void
2659 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
2660 {
2661         struct mem_cgroup *memcg;
2662         int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
2663
2664         if (!swapout) /* this was a swap cache but the swap is unused ! */
2665                 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2666
2667         memcg = __mem_cgroup_uncharge_common(page, ctype);
2668
2669         /*
2670          * record memcg information,  if swapout && memcg != NULL,
2671          * mem_cgroup_get() was called in uncharge().
2672          */
2673         if (do_swap_account && swapout && memcg)
2674                 swap_cgroup_record(ent, css_id(&memcg->css));
2675 }
2676 #endif
2677
2678 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2679 /*
2680  * called from swap_entry_free(). remove record in swap_cgroup and
2681  * uncharge "memsw" account.
2682  */
2683 void mem_cgroup_uncharge_swap(swp_entry_t ent)
2684 {
2685         struct mem_cgroup *memcg;
2686         unsigned short id;
2687
2688         if (!do_swap_account)
2689                 return;
2690
2691         id = swap_cgroup_record(ent, 0);
2692         rcu_read_lock();
2693         memcg = mem_cgroup_lookup(id);
2694         if (memcg) {
2695                 /*
2696                  * We uncharge this because swap is freed.
2697                  * This memcg can be obsolete one. We avoid calling css_tryget
2698                  */
2699                 if (!mem_cgroup_is_root(memcg))
2700                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2701                 mem_cgroup_swap_statistics(memcg, false);
2702                 mem_cgroup_put(memcg);
2703         }
2704         rcu_read_unlock();
2705 }
2706
2707 /**
2708  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2709  * @entry: swap entry to be moved
2710  * @from:  mem_cgroup which the entry is moved from
2711  * @to:  mem_cgroup which the entry is moved to
2712  * @need_fixup: whether we should fixup res_counters and refcounts.
2713  *
2714  * It succeeds only when the swap_cgroup's record for this entry is the same
2715  * as the mem_cgroup's id of @from.
2716  *
2717  * Returns 0 on success, -EINVAL on failure.
2718  *
2719  * The caller must have charged to @to, IOW, called res_counter_charge() about
2720  * both res and memsw, and called css_get().
2721  */
2722 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2723                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2724 {
2725         unsigned short old_id, new_id;
2726
2727         old_id = css_id(&from->css);
2728         new_id = css_id(&to->css);
2729
2730         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2731                 mem_cgroup_swap_statistics(from, false);
2732                 mem_cgroup_swap_statistics(to, true);
2733                 /*
2734                  * This function is only called from task migration context now.
2735                  * It postpones res_counter and refcount handling till the end
2736                  * of task migration(mem_cgroup_clear_mc()) for performance
2737                  * improvement. But we cannot postpone mem_cgroup_get(to)
2738                  * because if the process that has been moved to @to does
2739                  * swap-in, the refcount of @to might be decreased to 0.
2740                  */
2741                 mem_cgroup_get(to);
2742                 if (need_fixup) {
2743                         if (!mem_cgroup_is_root(from))
2744                                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2745                         mem_cgroup_put(from);
2746                         /*
2747                          * we charged both to->res and to->memsw, so we should
2748                          * uncharge to->res.
2749                          */
2750                         if (!mem_cgroup_is_root(to))
2751                                 res_counter_uncharge(&to->res, PAGE_SIZE);
2752                 }
2753                 return 0;
2754         }
2755         return -EINVAL;
2756 }
2757 #else
2758 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2759                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2760 {
2761         return -EINVAL;
2762 }
2763 #endif
2764
2765 /*
2766  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2767  * page belongs to.
2768  */
2769 int mem_cgroup_prepare_migration(struct page *page,
2770         struct page *newpage, struct mem_cgroup **ptr)
2771 {
2772         struct page_cgroup *pc;
2773         struct mem_cgroup *mem = NULL;
2774         enum charge_type ctype;
2775         int ret = 0;
2776
2777         if (mem_cgroup_disabled())
2778                 return 0;
2779
2780         pc = lookup_page_cgroup(page);
2781         lock_page_cgroup(pc);
2782         if (PageCgroupUsed(pc)) {
2783                 mem = pc->mem_cgroup;
2784                 css_get(&mem->css);
2785                 /*
2786                  * At migrating an anonymous page, its mapcount goes down
2787                  * to 0 and uncharge() will be called. But, even if it's fully
2788                  * unmapped, migration may fail and this page has to be
2789                  * charged again. We set MIGRATION flag here and delay uncharge
2790                  * until end_migration() is called
2791                  *
2792                  * Corner Case Thinking
2793                  * A)
2794                  * When the old page was mapped as Anon and it's unmap-and-freed
2795                  * while migration was ongoing.
2796                  * If unmap finds the old page, uncharge() of it will be delayed
2797                  * until end_migration(). If unmap finds a new page, it's
2798                  * uncharged when it make mapcount to be 1->0. If unmap code
2799                  * finds swap_migration_entry, the new page will not be mapped
2800                  * and end_migration() will find it(mapcount==0).
2801                  *
2802                  * B)
2803                  * When the old page was mapped but migraion fails, the kernel
2804                  * remaps it. A charge for it is kept by MIGRATION flag even
2805                  * if mapcount goes down to 0. We can do remap successfully
2806                  * without charging it again.
2807                  *
2808                  * C)
2809                  * The "old" page is under lock_page() until the end of
2810                  * migration, so, the old page itself will not be swapped-out.
2811                  * If the new page is swapped out before end_migraton, our
2812                  * hook to usual swap-out path will catch the event.
2813                  */
2814                 if (PageAnon(page))
2815                         SetPageCgroupMigration(pc);
2816         }
2817         unlock_page_cgroup(pc);
2818         /*
2819          * If the page is not charged at this point,
2820          * we return here.
2821          */
2822         if (!mem)
2823                 return 0;
2824
2825         *ptr = mem;
2826         ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
2827         css_put(&mem->css);/* drop extra refcnt */
2828         if (ret || *ptr == NULL) {
2829                 if (PageAnon(page)) {
2830                         lock_page_cgroup(pc);
2831                         ClearPageCgroupMigration(pc);
2832                         unlock_page_cgroup(pc);
2833                         /*
2834                          * The old page may be fully unmapped while we kept it.
2835                          */
2836                         mem_cgroup_uncharge_page(page);
2837                 }
2838                 return -ENOMEM;
2839         }
2840         /*
2841          * We charge new page before it's used/mapped. So, even if unlock_page()
2842          * is called before end_migration, we can catch all events on this new
2843          * page. In the case new page is migrated but not remapped, new page's
2844          * mapcount will be finally 0 and we call uncharge in end_migration().
2845          */
2846         pc = lookup_page_cgroup(newpage);
2847         if (PageAnon(page))
2848                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2849         else if (page_is_file_cache(page))
2850                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2851         else
2852                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
2853         __mem_cgroup_commit_charge(mem, pc, ctype);
2854         return ret;
2855 }
2856
2857 /* remove redundant charge if migration failed*/
2858 void mem_cgroup_end_migration(struct mem_cgroup *mem,
2859         struct page *oldpage, struct page *newpage)
2860 {
2861         struct page *used, *unused;
2862         struct page_cgroup *pc;
2863
2864         if (!mem)
2865                 return;
2866         /* blocks rmdir() */
2867         cgroup_exclude_rmdir(&mem->css);
2868         /* at migration success, oldpage->mapping is NULL. */
2869         if (oldpage->mapping) {
2870                 used = oldpage;
2871                 unused = newpage;
2872         } else {
2873                 used = newpage;
2874                 unused = oldpage;
2875         }
2876         /*
2877          * We disallowed uncharge of pages under migration because mapcount
2878          * of the page goes down to zero, temporarly.
2879          * Clear the flag and check the page should be charged.
2880          */
2881         pc = lookup_page_cgroup(oldpage);
2882         lock_page_cgroup(pc);
2883         ClearPageCgroupMigration(pc);
2884         unlock_page_cgroup(pc);
2885
2886         __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2887
2888         /*
2889          * If a page is a file cache, radix-tree replacement is very atomic
2890          * and we can skip this check. When it was an Anon page, its mapcount
2891          * goes down to 0. But because we added MIGRATION flage, it's not
2892          * uncharged yet. There are several case but page->mapcount check
2893          * and USED bit check in mem_cgroup_uncharge_page() will do enough
2894          * check. (see prepare_charge() also)
2895          */
2896         if (PageAnon(used))
2897                 mem_cgroup_uncharge_page(used);
2898         /*
2899          * At migration, we may charge account against cgroup which has no
2900          * tasks.
2901          * So, rmdir()->pre_destroy() can be called while we do this charge.
2902          * In that case, we need to call pre_destroy() again. check it here.
2903          */
2904         cgroup_release_and_wakeup_rmdir(&mem->css);
2905 }
2906
2907 /*
2908  * A call to try to shrink memory usage on charge failure at shmem's swapin.
2909  * Calling hierarchical_reclaim is not enough because we should update
2910  * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2911  * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2912  * not from the memcg which this page would be charged to.
2913  * try_charge_swapin does all of these works properly.
2914  */
2915 int mem_cgroup_shmem_charge_fallback(struct page *page,
2916                             struct mm_struct *mm,
2917                             gfp_t gfp_mask)
2918 {
2919         struct mem_cgroup *mem = NULL;
2920         int ret;
2921
2922         if (mem_cgroup_disabled())
2923                 return 0;
2924
2925         ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2926         if (!ret)
2927                 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
2928
2929         return ret;
2930 }
2931
2932 static DEFINE_MUTEX(set_limit_mutex);
2933
2934 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2935                                 unsigned long long val)
2936 {
2937         int retry_count;
2938         u64 memswlimit, memlimit;
2939         int ret = 0;
2940         int children = mem_cgroup_count_children(memcg);
2941         u64 curusage, oldusage;
2942         int enlarge;
2943
2944         /*
2945          * For keeping hierarchical_reclaim simple, how long we should retry
2946          * is depends on callers. We set our retry-count to be function
2947          * of # of children which we should visit in this loop.
2948          */
2949         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
2950
2951         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2952
2953         enlarge = 0;
2954         while (retry_count) {
2955                 if (signal_pending(current)) {
2956                         ret = -EINTR;
2957                         break;
2958                 }
2959                 /*
2960                  * Rather than hide all in some function, I do this in
2961                  * open coded manner. You see what this really does.
2962                  * We have to guarantee mem->res.limit < mem->memsw.limit.
2963                  */
2964                 mutex_lock(&set_limit_mutex);
2965                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2966                 if (memswlimit < val) {
2967                         ret = -EINVAL;
2968                         mutex_unlock(&set_limit_mutex);
2969                         break;
2970                 }
2971
2972                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2973                 if (memlimit < val)
2974                         enlarge = 1;
2975
2976                 ret = res_counter_set_limit(&memcg->res, val);
2977                 if (!ret) {
2978                         if (memswlimit == val)
2979                                 memcg->memsw_is_minimum = true;
2980                         else
2981                                 memcg->memsw_is_minimum = false;
2982                 }
2983                 mutex_unlock(&set_limit_mutex);
2984
2985                 if (!ret)
2986                         break;
2987
2988                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
2989                                                 MEM_CGROUP_RECLAIM_SHRINK);
2990                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2991                 /* Usage is reduced ? */
2992                 if (curusage >= oldusage)
2993                         retry_count--;
2994                 else
2995                         oldusage = curusage;
2996         }
2997         if (!ret && enlarge)
2998                 memcg_oom_recover(memcg);
2999
3000         return ret;
3001 }
3002
3003 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3004                                         unsigned long long val)
3005 {
3006         int retry_count;
3007         u64 memlimit, memswlimit, oldusage, curusage;
3008         int children = mem_cgroup_count_children(memcg);
3009         int ret = -EBUSY;
3010         int enlarge = 0;
3011
3012         /* see mem_cgroup_resize_res_limit */
3013         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
3014         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3015         while (retry_count) {
3016                 if (signal_pending(current)) {
3017                         ret = -EINTR;
3018                         break;
3019                 }
3020                 /*
3021                  * Rather than hide all in some function, I do this in
3022                  * open coded manner. You see what this really does.
3023                  * We have to guarantee mem->res.limit < mem->memsw.limit.
3024                  */
3025                 mutex_lock(&set_limit_mutex);
3026                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3027                 if (memlimit > val) {
3028                         ret = -EINVAL;
3029                         mutex_unlock(&set_limit_mutex);
3030                         break;
3031                 }
3032                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3033                 if (memswlimit < val)
3034                         enlarge = 1;
3035                 ret = res_counter_set_limit(&memcg->memsw, val);
3036                 if (!ret) {
3037                         if (memlimit == val)
3038                                 memcg->memsw_is_minimum = true;
3039                         else
3040                                 memcg->memsw_is_minimum = false;
3041                 }
3042                 mutex_unlock(&set_limit_mutex);
3043
3044                 if (!ret)
3045                         break;
3046
3047                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3048                                                 MEM_CGROUP_RECLAIM_NOSWAP |
3049                                                 MEM_CGROUP_RECLAIM_SHRINK);
3050                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3051                 /* Usage is reduced ? */
3052                 if (curusage >= oldusage)
3053                         retry_count--;
3054                 else
3055                         oldusage = curusage;
3056         }
3057         if (!ret && enlarge)
3058                 memcg_oom_recover(memcg);
3059         return ret;
3060 }
3061
3062 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3063                                             gfp_t gfp_mask)
3064 {
3065         unsigned long nr_reclaimed = 0;
3066         struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3067         unsigned long reclaimed;
3068         int loop = 0;
3069         struct mem_cgroup_tree_per_zone *mctz;
3070         unsigned long long excess;
3071
3072         if (order > 0)
3073                 return 0;
3074
3075         mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3076         /*
3077          * This loop can run a while, specially if mem_cgroup's continuously
3078          * keep exceeding their soft limit and putting the system under
3079          * pressure
3080          */
3081         do {
3082                 if (next_mz)
3083                         mz = next_mz;
3084                 else
3085                         mz = mem_cgroup_largest_soft_limit_node(mctz);
3086                 if (!mz)
3087                         break;
3088
3089                 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
3090                                                 gfp_mask,
3091                                                 MEM_CGROUP_RECLAIM_SOFT);
3092                 nr_reclaimed += reclaimed;
3093                 spin_lock(&mctz->lock);
3094
3095                 /*
3096                  * If we failed to reclaim anything from this memory cgroup
3097                  * it is time to move on to the next cgroup
3098                  */
3099                 next_mz = NULL;
3100                 if (!reclaimed) {
3101                         do {
3102                                 /*
3103                                  * Loop until we find yet another one.
3104                                  *
3105                                  * By the time we get the soft_limit lock
3106                                  * again, someone might have aded the
3107                                  * group back on the RB tree. Iterate to
3108                                  * make sure we get a different mem.
3109                                  * mem_cgroup_largest_soft_limit_node returns
3110                                  * NULL if no other cgroup is present on
3111                                  * the tree
3112                                  */
3113                                 next_mz =
3114                                 __mem_cgroup_largest_soft_limit_node(mctz);
3115                                 if (next_mz == mz) {
3116                                         css_put(&next_mz->mem->css);
3117                                         next_mz = NULL;
3118                                 } else /* next_mz == NULL or other memcg */
3119                                         break;
3120                         } while (1);
3121                 }
3122                 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
3123                 excess = res_counter_soft_limit_excess(&mz->mem->res);
3124                 /*
3125                  * One school of thought says that we should not add
3126                  * back the node to the tree if reclaim returns 0.
3127                  * But our reclaim could return 0, simply because due
3128                  * to priority we are exposing a smaller subset of
3129                  * memory to reclaim from. Consider this as a longer
3130                  * term TODO.
3131                  */
3132                 /* If excess == 0, no tree ops */
3133                 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
3134                 spin_unlock(&mctz->lock);
3135                 css_put(&mz->mem->css);
3136                 loop++;
3137                 /*
3138                  * Could not reclaim anything and there are no more
3139                  * mem cgroups to try or we seem to be looping without
3140                  * reclaiming anything.
3141                  */
3142                 if (!nr_reclaimed &&
3143                         (next_mz == NULL ||
3144                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3145                         break;
3146         } while (!nr_reclaimed);
3147         if (next_mz)
3148                 css_put(&next_mz->mem->css);
3149         return nr_reclaimed;
3150 }
3151
3152 /*
3153  * This routine traverse page_cgroup in given list and drop them all.
3154  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
3155  */
3156 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
3157                                 int node, int zid, enum lru_list lru)
3158 {
3159         struct zone *zone;
3160         struct mem_cgroup_per_zone *mz;
3161         struct page_cgroup *pc, *busy;
3162         unsigned long flags, loop;
3163         struct list_head *list;
3164         int ret = 0;
3165
3166         zone = &NODE_DATA(node)->node_zones[zid];
3167         mz = mem_cgroup_zoneinfo(mem, node, zid);
3168         list = &mz->lists[lru];
3169
3170         loop = MEM_CGROUP_ZSTAT(mz, lru);
3171         /* give some margin against EBUSY etc...*/
3172         loop += 256;
3173         busy = NULL;
3174         while (loop--) {
3175                 ret = 0;
3176                 spin_lock_irqsave(&zone->lru_lock, flags);
3177                 if (list_empty(list)) {
3178                         spin_unlock_irqrestore(&zone->lru_lock, flags);
3179                         break;
3180                 }
3181                 pc = list_entry(list->prev, struct page_cgroup, lru);
3182                 if (busy == pc) {
3183                         list_move(&pc->lru, list);
3184                         busy = NULL;
3185                         spin_unlock_irqrestore(&zone->lru_lock, flags);
3186                         continue;
3187                 }
3188                 spin_unlock_irqrestore(&zone->lru_lock, flags);
3189
3190                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
3191                 if (ret == -ENOMEM)
3192                         break;
3193
3194                 if (ret == -EBUSY || ret == -EINVAL) {
3195                         /* found lock contention or "pc" is obsolete. */
3196                         busy = pc;
3197                         cond_resched();
3198                 } else
3199                         busy = NULL;
3200         }
3201
3202         if (!ret && !list_empty(list))
3203                 return -EBUSY;
3204         return ret;
3205 }
3206
3207 /*
3208  * make mem_cgroup's charge to be 0 if there is no task.
3209  * This enables deleting this mem_cgroup.
3210  */
3211 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
3212 {
3213         int ret;
3214         int node, zid, shrink;
3215         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
3216         struct cgroup *cgrp = mem->css.cgroup;
3217
3218         css_get(&mem->css);
3219
3220         shrink = 0;
3221         /* should free all ? */
3222         if (free_all)
3223                 goto try_to_free;
3224 move_account:
3225         do {
3226                 ret = -EBUSY;
3227                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
3228                         goto out;
3229                 ret = -EINTR;
3230                 if (signal_pending(current))
3231                         goto out;
3232                 /* This is for making all *used* pages to be on LRU. */
3233                 lru_add_drain_all();
3234                 drain_all_stock_sync();
3235                 ret = 0;
3236                 mem_cgroup_start_move(mem);
3237                 for_each_node_state(node, N_HIGH_MEMORY) {
3238                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
3239                                 enum lru_list l;
3240                                 for_each_lru(l) {
3241                                         ret = mem_cgroup_force_empty_list(mem,
3242                                                         node, zid, l);
3243                                         if (ret)
3244                                                 break;
3245                                 }
3246                         }
3247                         if (ret)
3248                                 break;
3249                 }
3250                 mem_cgroup_end_move(mem);
3251                 memcg_oom_recover(mem);
3252                 /* it seems parent cgroup doesn't have enough mem */
3253                 if (ret == -ENOMEM)
3254                         goto try_to_free;
3255                 cond_resched();
3256         /* "ret" should also be checked to ensure all lists are empty. */
3257         } while (mem->res.usage > 0 || ret);
3258 out:
3259         css_put(&mem->css);
3260         return ret;
3261
3262 try_to_free:
3263         /* returns EBUSY if there is a task or if we come here twice. */
3264         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
3265                 ret = -EBUSY;
3266                 goto out;
3267         }
3268         /* we call try-to-free pages for make this cgroup empty */
3269         lru_add_drain_all();
3270         /* try to free all pages in this cgroup */
3271         shrink = 1;
3272         while (nr_retries && mem->res.usage > 0) {
3273                 int progress;
3274
3275                 if (signal_pending(current)) {
3276                         ret = -EINTR;
3277                         goto out;
3278                 }
3279                 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3280                                                 false, get_swappiness(mem));
3281                 if (!progress) {
3282                         nr_retries--;
3283                         /* maybe some writeback is necessary */
3284                         congestion_wait(BLK_RW_ASYNC, HZ/10);
3285                 }
3286
3287         }
3288         lru_add_drain();
3289         /* try move_account...there may be some *locked* pages. */
3290         goto move_account;
3291 }
3292
3293 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3294 {
3295         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3296 }
3297
3298
3299 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3300 {
3301         return mem_cgroup_from_cont(cont)->use_hierarchy;
3302 }
3303
3304 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3305                                         u64 val)
3306 {
3307         int retval = 0;
3308         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3309         struct cgroup *parent = cont->parent;
3310         struct mem_cgroup *parent_mem = NULL;
3311
3312         if (parent)
3313                 parent_mem = mem_cgroup_from_cont(parent);
3314
3315         cgroup_lock();
3316         /*
3317          * If parent's use_hierarchy is set, we can't make any modifications
3318          * in the child subtrees. If it is unset, then the change can
3319          * occur, provided the current cgroup has no children.
3320          *
3321          * For the root cgroup, parent_mem is NULL, we allow value to be
3322          * set if there are no children.
3323          */
3324         if ((!parent_mem || !parent_mem->use_hierarchy) &&
3325                                 (val == 1 || val == 0)) {
3326                 if (list_empty(&cont->children))
3327                         mem->use_hierarchy = val;
3328                 else
3329                         retval = -EBUSY;
3330         } else
3331                 retval = -EINVAL;
3332         cgroup_unlock();
3333
3334         return retval;
3335 }
3336
3337
3338 static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3339                                 enum mem_cgroup_stat_index idx)
3340 {
3341         struct mem_cgroup *iter;
3342         s64 val = 0;
3343
3344         /* each per cpu's value can be minus.Then, use s64 */
3345         for_each_mem_cgroup_tree(iter, mem)
3346                 val += mem_cgroup_read_stat(iter, idx);
3347
3348         if (val < 0) /* race ? */
3349                 val = 0;
3350         return val;
3351 }
3352
3353 static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3354 {
3355         u64 val;
3356
3357         if (!mem_cgroup_is_root(mem)) {
3358                 if (!swap)
3359                         return res_counter_read_u64(&mem->res, RES_USAGE);
3360                 else
3361                         return res_counter_read_u64(&mem->memsw, RES_USAGE);
3362         }
3363
3364         val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
3365         val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
3366
3367         if (swap)
3368                 val += mem_cgroup_get_recursive_idx_stat(mem,
3369                                 MEM_CGROUP_STAT_SWAPOUT);
3370
3371         return val << PAGE_SHIFT;
3372 }
3373
3374 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
3375 {
3376         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3377         u64 val;
3378         int type, name;
3379
3380         type = MEMFILE_TYPE(cft->private);
3381         name = MEMFILE_ATTR(cft->private);
3382         switch (type) {
3383         case _MEM:
3384                 if (name == RES_USAGE)
3385                         val = mem_cgroup_usage(mem, false);
3386                 else
3387                         val = res_counter_read_u64(&mem->res, name);
3388                 break;
3389         case _MEMSWAP:
3390                 if (name == RES_USAGE)
3391                         val = mem_cgroup_usage(mem, true);
3392                 else
3393                         val = res_counter_read_u64(&mem->memsw, name);
3394                 break;
3395         default:
3396                 BUG();
3397                 break;
3398         }
3399         return val;
3400 }
3401 /*
3402  * The user of this function is...
3403  * RES_LIMIT.
3404  */
3405 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3406                             const char *buffer)
3407 {
3408         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3409         int type, name;
3410         unsigned long long val;
3411         int ret;
3412
3413         type = MEMFILE_TYPE(cft->private);
3414         name = MEMFILE_ATTR(cft->private);
3415         switch (name) {
3416         case RES_LIMIT:
3417                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3418                         ret = -EINVAL;
3419                         break;
3420                 }
3421                 /* This function does all necessary parse...reuse it */
3422                 ret = res_counter_memparse_write_strategy(buffer, &val);
3423                 if (ret)
3424                         break;
3425                 if (type == _MEM)
3426                         ret = mem_cgroup_resize_limit(memcg, val);
3427                 else
3428                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
3429                 break;
3430         case RES_SOFT_LIMIT:
3431                 ret = res_counter_memparse_write_strategy(buffer, &val);
3432                 if (ret)
3433                         break;
3434                 /*
3435                  * For memsw, soft limits are hard to implement in terms
3436                  * of semantics, for now, we support soft limits for
3437                  * control without swap
3438                  */
3439                 if (type == _MEM)
3440                         ret = res_counter_set_soft_limit(&memcg->res, val);
3441                 else
3442                         ret = -EINVAL;
3443                 break;
3444         default:
3445                 ret = -EINVAL; /* should be BUG() ? */
3446                 break;
3447         }
3448         return ret;
3449 }
3450
3451 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3452                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3453 {
3454         struct cgroup *cgroup;
3455         unsigned long long min_limit, min_memsw_limit, tmp;
3456
3457         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3458         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3459         cgroup = memcg->css.cgroup;
3460         if (!memcg->use_hierarchy)
3461                 goto out;
3462
3463         while (cgroup->parent) {
3464                 cgroup = cgroup->parent;
3465                 memcg = mem_cgroup_from_cont(cgroup);
3466                 if (!memcg->use_hierarchy)
3467                         break;
3468                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3469                 min_limit = min(min_limit, tmp);
3470                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3471                 min_memsw_limit = min(min_memsw_limit, tmp);
3472         }
3473 out:
3474         *mem_limit = min_limit;
3475         *memsw_limit = min_memsw_limit;
3476         return;
3477 }
3478
3479 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
3480 {
3481         struct mem_cgroup *mem;
3482         int type, name;
3483
3484         mem = mem_cgroup_from_cont(cont);
3485         type = MEMFILE_TYPE(event);
3486         name = MEMFILE_ATTR(event);
3487         switch (name) {
3488         case RES_MAX_USAGE:
3489                 if (type == _MEM)
3490                         res_counter_reset_max(&mem->res);
3491                 else
3492                         res_counter_reset_max(&mem->memsw);
3493                 break;
3494         case RES_FAILCNT:
3495                 if (type == _MEM)
3496                         res_counter_reset_failcnt(&mem->res);
3497                 else
3498                         res_counter_reset_failcnt(&mem->memsw);
3499                 break;
3500         }
3501
3502         return 0;
3503 }
3504
3505 static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3506                                         struct cftype *cft)
3507 {
3508         return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3509 }
3510
3511 #ifdef CONFIG_MMU
3512 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3513                                         struct cftype *cft, u64 val)
3514 {
3515         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3516
3517         if (val >= (1 << NR_MOVE_TYPE))
3518                 return -EINVAL;
3519         /*
3520          * We check this value several times in both in can_attach() and
3521          * attach(), so we need cgroup lock to prevent this value from being
3522          * inconsistent.
3523          */
3524         cgroup_lock();
3525         mem->move_charge_at_immigrate = val;
3526         cgroup_unlock();
3527
3528         return 0;
3529 }
3530 #else
3531 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3532                                         struct cftype *cft, u64 val)
3533 {
3534         return -ENOSYS;
3535 }
3536 #endif
3537
3538
3539 /* For read statistics */
3540 enum {
3541         MCS_CACHE,
3542         MCS_RSS,
3543         MCS_FILE_MAPPED,
3544         MCS_PGPGIN,
3545         MCS_PGPGOUT,
3546         MCS_SWAP,
3547         MCS_INACTIVE_ANON,
3548         MCS_ACTIVE_ANON,
3549         MCS_INACTIVE_FILE,
3550         MCS_ACTIVE_FILE,
3551         MCS_UNEVICTABLE,
3552         NR_MCS_STAT,
3553 };
3554
3555 struct mcs_total_stat {
3556         s64 stat[NR_MCS_STAT];
3557 };
3558
3559 struct {
3560         char *local_name;
3561         char *total_name;
3562 } memcg_stat_strings[NR_MCS_STAT] = {
3563         {"cache", "total_cache"},
3564         {"rss", "total_rss"},
3565         {"mapped_file", "total_mapped_file"},
3566         {"pgpgin", "total_pgpgin"},
3567         {"pgpgout", "total_pgpgout"},
3568         {"swap", "total_swap"},
3569         {"inactive_anon", "total_inactive_anon"},
3570         {"active_anon", "total_active_anon"},
3571         {"inactive_file", "total_inactive_file"},
3572         {"active_file", "total_active_file"},
3573         {"unevictable", "total_unevictable"}
3574 };
3575
3576
3577 static void
3578 mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3579 {
3580         s64 val;
3581
3582         /* per cpu stat */
3583         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
3584         s->stat[MCS_CACHE] += val * PAGE_SIZE;
3585         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
3586         s->stat[MCS_RSS] += val * PAGE_SIZE;
3587         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
3588         s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
3589         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
3590         s->stat[MCS_PGPGIN] += val;
3591         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
3592         s->stat[MCS_PGPGOUT] += val;
3593         if (do_swap_account) {
3594                 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
3595                 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3596         }
3597
3598         /* per zone stat */
3599         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3600         s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3601         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3602         s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3603         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3604         s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3605         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3606         s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3607         val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3608         s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
3609 }
3610
3611 static void
3612 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3613 {
3614         struct mem_cgroup *iter;
3615
3616         for_each_mem_cgroup_tree(iter, mem)
3617                 mem_cgroup_get_local_stat(iter, s);
3618 }
3619
3620 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3621                                  struct cgroup_map_cb *cb)
3622 {
3623         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
3624         struct mcs_total_stat mystat;
3625         int i;
3626
3627         memset(&mystat, 0, sizeof(mystat));
3628         mem_cgroup_get_local_stat(mem_cont, &mystat);
3629
3630         for (i = 0; i < NR_MCS_STAT; i++) {
3631                 if (i == MCS_SWAP && !do_swap_account)
3632                         continue;
3633                 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
3634         }
3635
3636         /* Hierarchical information */
3637         {
3638                 unsigned long long limit, memsw_limit;
3639                 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3640                 cb->fill(cb, "hierarchical_memory_limit", limit);
3641                 if (do_swap_account)
3642                         cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3643         }
3644
3645         memset(&mystat, 0, sizeof(mystat));
3646         mem_cgroup_get_total_stat(mem_cont, &mystat);
3647         for (i = 0; i < NR_MCS_STAT; i++) {
3648                 if (i == MCS_SWAP && !do_swap_account)
3649                         continue;
3650                 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
3651         }
3652
3653 #ifdef CONFIG_DEBUG_VM
3654         cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
3655
3656         {
3657                 int nid, zid;
3658                 struct mem_cgroup_per_zone *mz;
3659                 unsigned long recent_rotated[2] = {0, 0};
3660                 unsigned long recent_scanned[2] = {0, 0};
3661
3662                 for_each_online_node(nid)
3663                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3664                                 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3665
3666                                 recent_rotated[0] +=
3667                                         mz->reclaim_stat.recent_rotated[0];
3668                                 recent_rotated[1] +=
3669                                         mz->reclaim_stat.recent_rotated[1];
3670                                 recent_scanned[0] +=
3671                                         mz->reclaim_stat.recent_scanned[0];
3672                                 recent_scanned[1] +=
3673                                         mz->reclaim_stat.recent_scanned[1];
3674                         }
3675                 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3676                 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3677                 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3678                 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3679         }
3680 #endif
3681
3682         return 0;
3683 }
3684
3685 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3686 {
3687         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3688
3689         return get_swappiness(memcg);
3690 }
3691
3692 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3693                                        u64 val)
3694 {
3695         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3696         struct mem_cgroup *parent;
3697
3698         if (val > 100)
3699                 return -EINVAL;
3700
3701         if (cgrp->parent == NULL)
3702                 return -EINVAL;
3703
3704         parent = mem_cgroup_from_cont(cgrp->parent);
3705
3706         cgroup_lock();
3707
3708         /* If under hierarchy, only empty-root can set this value */
3709         if ((parent->use_hierarchy) ||
3710             (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3711                 cgroup_unlock();
3712                 return -EINVAL;
3713         }
3714
3715         spin_lock(&memcg->reclaim_param_lock);
3716         memcg->swappiness = val;
3717         spin_unlock(&memcg->reclaim_param_lock);
3718
3719         cgroup_unlock();
3720
3721         return 0;
3722 }
3723
3724 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3725 {
3726         struct mem_cgroup_threshold_ary *t;
3727         u64 usage;
3728         int i;
3729
3730         rcu_read_lock();
3731         if (!swap)
3732                 t = rcu_dereference(memcg->thresholds.primary);
3733         else
3734                 t = rcu_dereference(memcg->memsw_thresholds.primary);
3735
3736         if (!t)
3737                 goto unlock;
3738
3739         usage = mem_cgroup_usage(memcg, swap);
3740
3741         /*
3742          * current_threshold points to threshold just below usage.
3743          * If it's not true, a threshold was crossed after last
3744          * call of __mem_cgroup_threshold().
3745          */
3746         i = t->current_threshold;
3747
3748         /*
3749          * Iterate backward over array of thresholds starting from
3750          * current_threshold and check if a threshold is crossed.
3751          * If none of thresholds below usage is crossed, we read
3752          * only one element of the array here.
3753          */
3754         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3755                 eventfd_signal(t->entries[i].eventfd, 1);
3756
3757         /* i = current_threshold + 1 */
3758         i++;
3759
3760         /*
3761          * Iterate forward over array of thresholds starting from
3762          * current_threshold+1 and check if a threshold is crossed.
3763          * If none of thresholds above usage is crossed, we read
3764          * only one element of the array here.
3765          */
3766         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3767                 eventfd_signal(t->entries[i].eventfd, 1);
3768
3769         /* Update current_threshold */
3770         t->current_threshold = i - 1;
3771 unlock:
3772         rcu_read_unlock();
3773 }
3774
3775 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3776 {
3777         while (memcg) {
3778                 __mem_cgroup_threshold(memcg, false);
3779                 if (do_swap_account)
3780                         __mem_cgroup_threshold(memcg, true);
3781
3782                 memcg = parent_mem_cgroup(memcg);
3783         }
3784 }
3785
3786 static int compare_thresholds(const void *a, const void *b)
3787 {
3788         const struct mem_cgroup_threshold *_a = a;
3789         const struct mem_cgroup_threshold *_b = b;
3790
3791         return _a->threshold - _b->threshold;
3792 }
3793
3794 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
3795 {
3796         struct mem_cgroup_eventfd_list *ev;
3797
3798         list_for_each_entry(ev, &mem->oom_notify, list)
3799                 eventfd_signal(ev->eventfd, 1);
3800         return 0;
3801 }
3802
3803 static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3804 {
3805         struct mem_cgroup *iter;
3806
3807         for_each_mem_cgroup_tree(iter, mem)
3808                 mem_cgroup_oom_notify_cb(iter);
3809 }
3810
3811 static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3812         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3813 {
3814         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3815         struct mem_cgroup_thresholds *thresholds;
3816         struct mem_cgroup_threshold_ary *new;
3817         int type = MEMFILE_TYPE(cft->private);
3818         u64 threshold, usage;
3819         int i, size, ret;
3820
3821         ret = res_counter_memparse_write_strategy(args, &threshold);
3822         if (ret)
3823                 return ret;
3824
3825         mutex_lock(&memcg->thresholds_lock);
3826
3827         if (type == _MEM)
3828                 thresholds = &memcg->thresholds;
3829         else if (type == _MEMSWAP)
3830                 thresholds = &memcg->memsw_thresholds;
3831         else
3832                 BUG();
3833
3834         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3835
3836         /* Check if a threshold crossed before adding a new one */
3837         if (thresholds->primary)
3838                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3839
3840         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3841
3842         /* Allocate memory for new array of thresholds */
3843         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3844                         GFP_KERNEL);
3845         if (!new) {
3846                 ret = -ENOMEM;
3847                 goto unlock;
3848         }
3849         new->size = size;
3850
3851         /* Copy thresholds (if any) to new array */
3852         if (thresholds->primary) {
3853                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3854                                 sizeof(struct mem_cgroup_threshold));
3855         }
3856
3857         /* Add new threshold */
3858         new->entries[size - 1].eventfd = eventfd;
3859         new->entries[size - 1].threshold = threshold;
3860
3861         /* Sort thresholds. Registering of new threshold isn't time-critical */
3862         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3863                         compare_thresholds, NULL);
3864
3865         /* Find current threshold */
3866         new->current_threshold = -1;
3867         for (i = 0; i < size; i++) {
3868                 if (new->entries[i].threshold < usage) {
3869                         /*
3870                          * new->current_threshold will not be used until
3871                          * rcu_assign_pointer(), so it's safe to increment
3872                          * it here.
3873                          */
3874                         ++new->current_threshold;
3875                 }
3876         }
3877
3878         /* Free old spare buffer and save old primary buffer as spare */
3879         kfree(thresholds->spare);
3880         thresholds->spare = thresholds->primary;
3881
3882         rcu_assign_pointer(thresholds->primary, new);
3883
3884         /* To be sure that nobody uses thresholds */
3885         synchronize_rcu();
3886
3887 unlock:
3888         mutex_unlock(&memcg->thresholds_lock);
3889
3890         return ret;
3891 }
3892
3893 static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
3894         struct cftype *cft, struct eventfd_ctx *eventfd)
3895 {
3896         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3897         struct mem_cgroup_thresholds *thresholds;
3898         struct mem_cgroup_threshold_ary *new;
3899         int type = MEMFILE_TYPE(cft->private);
3900         u64 usage;
3901         int i, j, size;
3902
3903         mutex_lock(&memcg->thresholds_lock);
3904         if (type == _MEM)
3905                 thresholds = &memcg->thresholds;
3906         else if (type == _MEMSWAP)
3907                 thresholds = &memcg->memsw_thresholds;
3908         else
3909                 BUG();
3910
3911         /*
3912          * Something went wrong if we trying to unregister a threshold
3913          * if we don't have thresholds
3914          */
3915         BUG_ON(!thresholds);
3916
3917         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3918
3919         /* Check if a threshold crossed before removing */
3920         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3921
3922         /* Calculate new number of threshold */
3923         size = 0;
3924         for (i = 0; i < thresholds->primary->size; i++) {
3925                 if (thresholds->primary->entries[i].eventfd != eventfd)
3926                         size++;
3927         }
3928
3929         new = thresholds->spare;
3930
3931         /* Set thresholds array to NULL if we don't have thresholds */
3932         if (!size) {
3933                 kfree(new);
3934                 new = NULL;
3935                 goto swap_buffers;
3936         }
3937
3938         new->size = size;
3939
3940         /* Copy thresholds and find current threshold */
3941         new->current_threshold = -1;
3942         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3943                 if (thresholds->primary->entries[i].eventfd == eventfd)
3944                         continue;
3945
3946                 new->entries[j] = thresholds->primary->entries[i];
3947                 if (new->entries[j].threshold < usage) {
3948                         /*
3949                          * new->current_threshold will not be used
3950                          * until rcu_assign_pointer(), so it's safe to increment
3951                          * it here.
3952                          */
3953                         ++new->current_threshold;
3954                 }
3955                 j++;
3956         }
3957
3958 swap_buffers:
3959         /* Swap primary and spare array */
3960         thresholds->spare = thresholds->primary;
3961         rcu_assign_pointer(thresholds->primary, new);
3962
3963         /* To be sure that nobody uses thresholds */
3964         synchronize_rcu();
3965
3966         mutex_unlock(&memcg->thresholds_lock);
3967 }
3968
3969 static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
3970         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3971 {
3972         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3973         struct mem_cgroup_eventfd_list *event;
3974         int type = MEMFILE_TYPE(cft->private);
3975
3976         BUG_ON(type != _OOM_TYPE);
3977         event = kmalloc(sizeof(*event), GFP_KERNEL);
3978         if (!event)
3979                 return -ENOMEM;
3980
3981         mutex_lock(&memcg_oom_mutex);
3982
3983         event->eventfd = eventfd;
3984         list_add(&event->list, &memcg->oom_notify);
3985
3986         /* already in OOM ? */
3987         if (atomic_read(&memcg->oom_lock))
3988                 eventfd_signal(eventfd, 1);
3989         mutex_unlock(&memcg_oom_mutex);
3990
3991         return 0;
3992 }
3993
3994 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
3995         struct cftype *cft, struct eventfd_ctx *eventfd)
3996 {
3997         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3998         struct mem_cgroup_eventfd_list *ev, *tmp;
3999         int type = MEMFILE_TYPE(cft->private);
4000
4001         BUG_ON(type != _OOM_TYPE);
4002
4003         mutex_lock(&memcg_oom_mutex);
4004
4005         list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
4006                 if (ev->eventfd == eventfd) {
4007                         list_del(&ev->list);
4008                         kfree(ev);
4009                 }
4010         }
4011
4012         mutex_unlock(&memcg_oom_mutex);
4013 }
4014
4015 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
4016         struct cftype *cft,  struct cgroup_map_cb *cb)
4017 {
4018         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4019
4020         cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
4021
4022         if (atomic_read(&mem->oom_lock))
4023                 cb->fill(cb, "under_oom", 1);
4024         else
4025                 cb->fill(cb, "under_oom", 0);
4026         return 0;
4027 }
4028
4029 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
4030         struct cftype *cft, u64 val)
4031 {
4032         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4033         struct mem_cgroup *parent;
4034
4035         /* cannot set to root cgroup and only 0 and 1 are allowed */
4036         if (!cgrp->parent || !((val == 0) || (val == 1)))
4037                 return -EINVAL;
4038
4039         parent = mem_cgroup_from_cont(cgrp->parent);
4040
4041         cgroup_lock();
4042         /* oom-kill-disable is a flag for subhierarchy. */
4043         if ((parent->use_hierarchy) ||
4044             (mem->use_hierarchy && !list_empty(&cgrp->children))) {
4045                 cgroup_unlock();
4046                 return -EINVAL;
4047         }
4048         mem->oom_kill_disable = val;
4049         if (!val)
4050                 memcg_oom_recover(mem);
4051         cgroup_unlock();
4052         return 0;
4053 }
4054
4055 static struct cftype mem_cgroup_files[] = {
4056         {
4057                 .name = "usage_in_bytes",
4058                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4059                 .read_u64 = mem_cgroup_read,
4060                 .register_event = mem_cgroup_usage_register_event,
4061                 .unregister_event = mem_cgroup_usage_unregister_event,
4062         },
4063         {
4064                 .name = "max_usage_in_bytes",
4065                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4066                 .trigger = mem_cgroup_reset,
4067                 .read_u64 = mem_cgroup_read,
4068         },
4069         {
4070                 .name = "limit_in_bytes",
4071                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4072                 .write_string = mem_cgroup_write,
4073                 .read_u64 = mem_cgroup_read,
4074         },
4075         {
4076                 .name = "soft_limit_in_bytes",
4077                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4078                 .write_string = mem_cgroup_write,
4079                 .read_u64 = mem_cgroup_read,
4080         },
4081         {
4082                 .name = "failcnt",
4083                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4084                 .trigger = mem_cgroup_reset,
4085                 .read_u64 = mem_cgroup_read,
4086         },
4087         {
4088                 .name = "stat",
4089                 .read_map = mem_control_stat_show,
4090         },
4091         {
4092                 .name = "force_empty",
4093                 .trigger = mem_cgroup_force_empty_write,
4094         },
4095         {
4096                 .name = "use_hierarchy",
4097                 .write_u64 = mem_cgroup_hierarchy_write,
4098                 .read_u64 = mem_cgroup_hierarchy_read,
4099         },
4100         {
4101                 .name = "swappiness",
4102                 .read_u64 = mem_cgroup_swappiness_read,
4103                 .write_u64 = mem_cgroup_swappiness_write,
4104         },
4105         {
4106                 .name = "move_charge_at_immigrate",
4107                 .read_u64 = mem_cgroup_move_charge_read,
4108                 .write_u64 = mem_cgroup_move_charge_write,
4109         },
4110         {
4111                 .name = "oom_control",
4112                 .read_map = mem_cgroup_oom_control_read,
4113                 .write_u64 = mem_cgroup_oom_control_write,
4114                 .register_event = mem_cgroup_oom_register_event,
4115                 .unregister_event = mem_cgroup_oom_unregister_event,
4116                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4117         },
4118 };
4119
4120 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4121 static struct cftype memsw_cgroup_files[] = {
4122         {
4123                 .name = "memsw.usage_in_bytes",
4124                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4125                 .read_u64 = mem_cgroup_read,
4126                 .register_event = mem_cgroup_usage_register_event,
4127                 .unregister_event = mem_cgroup_usage_unregister_event,
4128         },
4129         {
4130                 .name = "memsw.max_usage_in_bytes",
4131                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
4132                 .trigger = mem_cgroup_reset,
4133                 .read_u64 = mem_cgroup_read,
4134         },
4135         {
4136                 .name = "memsw.limit_in_bytes",
4137                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
4138                 .write_string = mem_cgroup_write,
4139                 .read_u64 = mem_cgroup_read,
4140         },
4141         {
4142                 .name = "memsw.failcnt",
4143                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
4144                 .trigger = mem_cgroup_reset,
4145                 .read_u64 = mem_cgroup_read,
4146         },
4147 };
4148
4149 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4150 {
4151         if (!do_swap_account)
4152                 return 0;
4153         return cgroup_add_files(cont, ss, memsw_cgroup_files,
4154                                 ARRAY_SIZE(memsw_cgroup_files));
4155 };
4156 #else
4157 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4158 {
4159         return 0;
4160 }
4161 #endif
4162
4163 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4164 {
4165         struct mem_cgroup_per_node *pn;
4166         struct mem_cgroup_per_zone *mz;
4167         enum lru_list l;
4168         int zone, tmp = node;
4169         /*
4170          * This routine is called against possible nodes.
4171          * But it's BUG to call kmalloc() against offline node.
4172          *
4173          * TODO: this routine can waste much memory for nodes which will
4174          *       never be onlined. It's better to use memory hotplug callback
4175          *       function.
4176          */
4177         if (!node_state(node, N_NORMAL_MEMORY))
4178                 tmp = -1;
4179         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4180         if (!pn)
4181                 return 1;
4182
4183         mem->info.nodeinfo[node] = pn;
4184         memset(pn, 0, sizeof(*pn));
4185
4186         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4187                 mz = &pn->zoneinfo[zone];
4188                 for_each_lru(l)
4189                         INIT_LIST_HEAD(&mz->lists[l]);
4190                 mz->usage_in_excess = 0;
4191                 mz->on_tree = false;
4192                 mz->mem = mem;
4193         }
4194         return 0;
4195 }
4196
4197 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4198 {
4199         kfree(mem->info.nodeinfo[node]);
4200 }
4201
4202 static struct mem_cgroup *mem_cgroup_alloc(void)
4203 {
4204         struct mem_cgroup *mem;
4205         int size = sizeof(struct mem_cgroup);
4206
4207         /* Can be very big if MAX_NUMNODES is very big */
4208         if (size < PAGE_SIZE)
4209                 mem = kmalloc(size, GFP_KERNEL);
4210         else
4211                 mem = vmalloc(size);
4212
4213         if (!mem)
4214                 return NULL;
4215
4216         memset(mem, 0, size);
4217         mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4218         if (!mem->stat)
4219                 goto out_free;
4220         spin_lock_init(&mem->pcp_counter_lock);
4221         return mem;
4222
4223 out_free:
4224         if (size < PAGE_SIZE)
4225                 kfree(mem);
4226         else
4227                 vfree(mem);
4228         return NULL;
4229 }
4230
4231 /*
4232  * At destroying mem_cgroup, references from swap_cgroup can remain.
4233  * (scanning all at force_empty is too costly...)
4234  *
4235  * Instead of clearing all references at force_empty, we remember
4236  * the number of reference from swap_cgroup and free mem_cgroup when
4237  * it goes down to 0.
4238  *
4239  * Removal of cgroup itself succeeds regardless of refs from swap.
4240  */
4241
4242 static void __mem_cgroup_free(struct mem_cgroup *mem)
4243 {
4244         int node;
4245
4246         mem_cgroup_remove_from_trees(mem);
4247         free_css_id(&mem_cgroup_subsys, &mem->css);
4248
4249         for_each_node_state(node, N_POSSIBLE)
4250                 free_mem_cgroup_per_zone_info(mem, node);
4251
4252         free_percpu(mem->stat);
4253         if (sizeof(struct mem_cgroup) < PAGE_SIZE)
4254                 kfree(mem);
4255         else
4256                 vfree(mem);
4257 }
4258
4259 static void mem_cgroup_get(struct mem_cgroup *mem)
4260 {
4261         atomic_inc(&mem->refcnt);
4262 }
4263
4264 static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
4265 {
4266         if (atomic_sub_and_test(count, &mem->refcnt)) {
4267                 struct mem_cgroup *parent = parent_mem_cgroup(mem);
4268                 __mem_cgroup_free(mem);
4269                 if (parent)
4270                         mem_cgroup_put(parent);
4271         }
4272 }
4273
4274 static void mem_cgroup_put(struct mem_cgroup *mem)
4275 {
4276         __mem_cgroup_put(mem, 1);
4277 }
4278
4279 /*
4280  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4281  */
4282 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4283 {
4284         if (!mem->res.parent)
4285                 return NULL;
4286         return mem_cgroup_from_res_counter(mem->res.parent, res);
4287 }
4288
4289 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4290 static void __init enable_swap_cgroup(void)
4291 {
4292         if (!mem_cgroup_disabled() && really_do_swap_account)
4293                 do_swap_account = 1;
4294 }
4295 #else
4296 static void __init enable_swap_cgroup(void)
4297 {
4298 }
4299 #endif
4300
4301 static int mem_cgroup_soft_limit_tree_init(void)
4302 {
4303         struct mem_cgroup_tree_per_node *rtpn;
4304         struct mem_cgroup_tree_per_zone *rtpz;
4305         int tmp, node, zone;
4306
4307         for_each_node_state(node, N_POSSIBLE) {
4308                 tmp = node;
4309                 if (!node_state(node, N_NORMAL_MEMORY))
4310                         tmp = -1;
4311                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4312                 if (!rtpn)
4313                         return 1;
4314
4315                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4316
4317                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4318                         rtpz = &rtpn->rb_tree_per_zone[zone];
4319                         rtpz->rb_root = RB_ROOT;
4320                         spin_lock_init(&rtpz->lock);
4321                 }
4322         }
4323         return 0;
4324 }
4325
4326 static struct cgroup_subsys_state * __ref
4327 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4328 {
4329         struct mem_cgroup *mem, *parent;
4330         long error = -ENOMEM;
4331         int node;
4332
4333         mem = mem_cgroup_alloc();
4334         if (!mem)
4335                 return ERR_PTR(error);
4336
4337         for_each_node_state(node, N_POSSIBLE)
4338                 if (alloc_mem_cgroup_per_zone_info(mem, node))
4339                         goto free_out;
4340
4341         /* root ? */
4342         if (cont->parent == NULL) {
4343                 int cpu;
4344                 enable_swap_cgroup();
4345                 parent = NULL;
4346                 root_mem_cgroup = mem;
4347                 if (mem_cgroup_soft_limit_tree_init())
4348                         goto free_out;
4349                 for_each_possible_cpu(cpu) {
4350                         struct memcg_stock_pcp *stock =
4351                                                 &per_cpu(memcg_stock, cpu);
4352                         INIT_WORK(&stock->work, drain_local_stock);
4353                 }
4354                 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
4355         } else {
4356                 parent = mem_cgroup_from_cont(cont->parent);
4357                 mem->use_hierarchy = parent->use_hierarchy;
4358                 mem->oom_kill_disable = parent->oom_kill_disable;
4359         }
4360
4361         if (parent && parent->use_hierarchy) {
4362                 res_counter_init(&mem->res, &parent->res);
4363                 res_counter_init(&mem->memsw, &parent->memsw);
4364                 /*
4365                  * We increment refcnt of the parent to ensure that we can
4366                  * safely access it on res_counter_charge/uncharge.
4367                  * This refcnt will be decremented when freeing this
4368                  * mem_cgroup(see mem_cgroup_put).
4369                  */
4370                 mem_cgroup_get(parent);
4371         } else {
4372                 res_counter_init(&mem->res, NULL);
4373                 res_counter_init(&mem->memsw, NULL);
4374         }
4375         mem->last_scanned_child = 0;
4376         spin_lock_init(&mem->reclaim_param_lock);
4377         INIT_LIST_HEAD(&mem->oom_notify);
4378
4379         if (parent)
4380                 mem->swappiness = get_swappiness(parent);
4381         atomic_set(&mem->refcnt, 1);
4382         mem->move_charge_at_immigrate = 0;
4383         mutex_init(&mem->thresholds_lock);
4384         return &mem->css;
4385 free_out:
4386         __mem_cgroup_free(mem);
4387         root_mem_cgroup = NULL;
4388         return ERR_PTR(error);
4389 }
4390
4391 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
4392                                         struct cgroup *cont)
4393 {
4394         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4395
4396         return mem_cgroup_force_empty(mem, false);
4397 }
4398
4399 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4400                                 struct cgroup *cont)
4401 {
4402         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4403
4404         mem_cgroup_put(mem);
4405 }
4406
4407 static int mem_cgroup_populate(struct cgroup_subsys *ss,
4408                                 struct cgroup *cont)
4409 {
4410         int ret;
4411
4412         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4413                                 ARRAY_SIZE(mem_cgroup_files));
4414
4415         if (!ret)
4416                 ret = register_memsw_files(cont, ss);
4417         return ret;
4418 }
4419
4420 #ifdef CONFIG_MMU
4421 /* Handlers for move charge at task migration. */
4422 #define PRECHARGE_COUNT_AT_ONCE 256
4423 static int mem_cgroup_do_precharge(unsigned long count)
4424 {
4425         int ret = 0;
4426         int batch_count = PRECHARGE_COUNT_AT_ONCE;
4427         struct mem_cgroup *mem = mc.to;
4428
4429         if (mem_cgroup_is_root(mem)) {
4430                 mc.precharge += count;
4431                 /* we don't need css_get for root */
4432                 return ret;
4433         }
4434         /* try to charge at once */
4435         if (count > 1) {
4436                 struct res_counter *dummy;
4437                 /*
4438                  * "mem" cannot be under rmdir() because we've already checked
4439                  * by cgroup_lock_live_cgroup() that it is not removed and we
4440                  * are still under the same cgroup_mutex. So we can postpone
4441                  * css_get().
4442                  */
4443                 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4444                         goto one_by_one;
4445                 if (do_swap_account && res_counter_charge(&mem->memsw,
4446                                                 PAGE_SIZE * count, &dummy)) {
4447                         res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4448                         goto one_by_one;
4449                 }
4450                 mc.precharge += count;
4451                 return ret;
4452         }
4453 one_by_one:
4454         /* fall back to one by one charge */
4455         while (count--) {
4456                 if (signal_pending(current)) {
4457                         ret = -EINTR;
4458                         break;
4459                 }
4460                 if (!batch_count--) {
4461                         batch_count = PRECHARGE_COUNT_AT_ONCE;
4462                         cond_resched();
4463                 }
4464                 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
4465                 if (ret || !mem)
4466                         /* mem_cgroup_clear_mc() will do uncharge later */
4467                         return -ENOMEM;
4468                 mc.precharge++;
4469         }
4470         return ret;
4471 }
4472
4473 /**
4474  * is_target_pte_for_mc - check a pte whether it is valid for move charge
4475  * @vma: the vma the pte to be checked belongs
4476  * @addr: the address corresponding to the pte to be checked
4477  * @ptent: the pte to be checked
4478  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4479  *
4480  * Returns
4481  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4482  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4483  *     move charge. if @target is not NULL, the page is stored in target->page
4484  *     with extra refcnt got(Callers should handle it).
4485  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4486  *     target for charge migration. if @target is not NULL, the entry is stored
4487  *     in target->ent.
4488  *
4489  * Called with pte lock held.
4490  */
4491 union mc_target {
4492         struct page     *page;
4493         swp_entry_t     ent;
4494 };
4495
4496 enum mc_target_type {
4497         MC_TARGET_NONE, /* not used */
4498         MC_TARGET_PAGE,
4499         MC_TARGET_SWAP,
4500 };
4501
4502 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4503                                                 unsigned long addr, pte_t ptent)
4504 {
4505         struct page *page = vm_normal_page(vma, addr, ptent);
4506
4507         if (!page || !page_mapped(page))
4508                 return NULL;
4509         if (PageAnon(page)) {
4510                 /* we don't move shared anon */
4511                 if (!move_anon() || page_mapcount(page) > 2)
4512                         return NULL;
4513         } else if (!move_file())
4514                 /* we ignore mapcount for file pages */
4515                 return NULL;
4516         if (!get_page_unless_zero(page))
4517                 return NULL;
4518
4519         return page;
4520 }
4521
4522 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4523                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4524 {
4525         int usage_count;
4526         struct page *page = NULL;
4527         swp_entry_t ent = pte_to_swp_entry(ptent);
4528
4529         if (!move_anon() || non_swap_entry(ent))
4530                 return NULL;
4531         usage_count = mem_cgroup_count_swap_user(ent, &page);
4532         if (usage_count > 1) { /* we don't move shared anon */
4533                 if (page)
4534                         put_page(page);
4535                 return NULL;
4536         }
4537         if (do_swap_account)
4538                 entry->val = ent.val;
4539
4540         return page;
4541 }
4542
4543 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4544                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4545 {
4546         struct page *page = NULL;
4547         struct inode *inode;
4548         struct address_space *mapping;
4549         pgoff_t pgoff;
4550
4551         if (!vma->vm_file) /* anonymous vma */
4552                 return NULL;
4553         if (!move_file())
4554                 return NULL;
4555
4556         inode = vma->vm_file->f_path.dentry->d_inode;
4557         mapping = vma->vm_file->f_mapping;
4558         if (pte_none(ptent))
4559                 pgoff = linear_page_index(vma, addr);
4560         else /* pte_file(ptent) is true */
4561                 pgoff = pte_to_pgoff(ptent);
4562
4563         /* page is moved even if it's not RSS of this task(page-faulted). */
4564         if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4565                 page = find_get_page(mapping, pgoff);
4566         } else { /* shmem/tmpfs file. we should take account of swap too. */
4567                 swp_entry_t ent;
4568                 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4569                 if (do_swap_account)
4570                         entry->val = ent.val;
4571         }
4572
4573         return page;
4574 }
4575
4576 static int is_target_pte_for_mc(struct vm_area_struct *vma,
4577                 unsigned long addr, pte_t ptent, union mc_target *target)
4578 {
4579         struct page *page = NULL;
4580         struct page_cgroup *pc;
4581         int ret = 0;
4582         swp_entry_t ent = { .val = 0 };
4583
4584         if (pte_present(ptent))
4585                 page = mc_handle_present_pte(vma, addr, ptent);
4586         else if (is_swap_pte(ptent))
4587                 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4588         else if (pte_none(ptent) || pte_file(ptent))
4589                 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4590
4591         if (!page && !ent.val)
4592                 return 0;
4593         if (page) {
4594                 pc = lookup_page_cgroup(page);
4595                 /*
4596                  * Do only loose check w/o page_cgroup lock.
4597                  * mem_cgroup_move_account() checks the pc is valid or not under
4598                  * the lock.
4599                  */
4600                 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4601                         ret = MC_TARGET_PAGE;
4602                         if (target)
4603                                 target->page = page;
4604                 }
4605                 if (!ret || !target)
4606                         put_page(page);
4607         }
4608         /* There is a swap entry and a page doesn't exist or isn't charged */
4609         if (ent.val && !ret &&
4610                         css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4611                 ret = MC_TARGET_SWAP;
4612                 if (target)
4613                         target->ent = ent;
4614         }
4615         return ret;
4616 }
4617
4618 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4619                                         unsigned long addr, unsigned long end,
4620                                         struct mm_walk *walk)
4621 {
4622         struct vm_area_struct *vma = walk->private;
4623         pte_t *pte;
4624         spinlock_t *ptl;
4625
4626         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4627         for (; addr != end; pte++, addr += PAGE_SIZE)
4628                 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4629                         mc.precharge++; /* increment precharge temporarily */
4630         pte_unmap_unlock(pte - 1, ptl);
4631         cond_resched();
4632
4633         return 0;
4634 }
4635
4636 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4637 {
4638         unsigned long precharge;
4639         struct vm_area_struct *vma;
4640
4641         /* We've already held the mmap_sem */
4642         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4643                 struct mm_walk mem_cgroup_count_precharge_walk = {
4644                         .pmd_entry = mem_cgroup_count_precharge_pte_range,
4645                         .mm = mm,
4646                         .private = vma,
4647                 };
4648                 if (is_vm_hugetlb_page(vma))
4649                         continue;
4650                 walk_page_range(vma->vm_start, vma->vm_end,
4651                                         &mem_cgroup_count_precharge_walk);
4652         }
4653
4654         precharge = mc.precharge;
4655         mc.precharge = 0;
4656
4657         return precharge;
4658 }
4659
4660 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4661 {
4662         return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
4663 }
4664
4665 static void mem_cgroup_clear_mc(void)
4666 {
4667         struct mem_cgroup *from = mc.from;
4668         struct mem_cgroup *to = mc.to;
4669
4670         /* we must uncharge all the leftover precharges from mc.to */
4671         if (mc.precharge) {
4672                 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4673                 mc.precharge = 0;
4674         }
4675         /*
4676          * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4677          * we must uncharge here.
4678          */
4679         if (mc.moved_charge) {
4680                 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4681                 mc.moved_charge = 0;
4682         }
4683         /* we must fixup refcnts and charges */
4684         if (mc.moved_swap) {
4685                 /* uncharge swap account from the old cgroup */
4686                 if (!mem_cgroup_is_root(mc.from))
4687                         res_counter_uncharge(&mc.from->memsw,
4688                                                 PAGE_SIZE * mc.moved_swap);
4689                 __mem_cgroup_put(mc.from, mc.moved_swap);
4690
4691                 if (!mem_cgroup_is_root(mc.to)) {
4692                         /*
4693                          * we charged both to->res and to->memsw, so we should
4694                          * uncharge to->res.
4695                          */
4696                         res_counter_uncharge(&mc.to->res,
4697                                                 PAGE_SIZE * mc.moved_swap);
4698                 }
4699                 /* we've already done mem_cgroup_get(mc.to) */
4700
4701                 mc.moved_swap = 0;
4702         }
4703         if (mc.mm) {
4704                 up_read(&mc.mm->mmap_sem);
4705                 mmput(mc.mm);
4706         }
4707         spin_lock(&mc.lock);
4708         mc.from = NULL;
4709         mc.to = NULL;
4710         spin_unlock(&mc.lock);
4711         mc.moving_task = NULL;
4712         mc.mm = NULL;
4713         mem_cgroup_end_move(from);
4714         memcg_oom_recover(from);
4715         memcg_oom_recover(to);
4716         wake_up_all(&mc.waitq);
4717 }
4718
4719 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4720                                 struct cgroup *cgroup,
4721                                 struct task_struct *p,
4722                                 bool threadgroup)
4723 {
4724         int ret = 0;
4725         struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4726
4727         if (mem->move_charge_at_immigrate) {
4728                 struct mm_struct *mm;
4729                 struct mem_cgroup *from = mem_cgroup_from_task(p);
4730
4731                 VM_BUG_ON(from == mem);
4732
4733                 mm = get_task_mm(p);
4734                 if (!mm)
4735                         return 0;
4736                 /* We move charges only when we move a owner of the mm */
4737                 if (mm->owner == p) {
4738                         /*
4739                          * We do all the move charge works under one mmap_sem to
4740                          * avoid deadlock with down_write(&mmap_sem)
4741                          * -> try_charge() -> if (mc.moving_task) -> sleep.
4742                          */
4743                         down_read(&mm->mmap_sem);
4744
4745                         VM_BUG_ON(mc.from);
4746                         VM_BUG_ON(mc.to);
4747                         VM_BUG_ON(mc.precharge);
4748                         VM_BUG_ON(mc.moved_charge);
4749                         VM_BUG_ON(mc.moved_swap);
4750                         VM_BUG_ON(mc.moving_task);
4751                         VM_BUG_ON(mc.mm);
4752
4753                         mem_cgroup_start_move(from);
4754                         spin_lock(&mc.lock);
4755                         mc.from = from;
4756                         mc.to = mem;
4757                         mc.precharge = 0;
4758                         mc.moved_charge = 0;
4759                         mc.moved_swap = 0;
4760                         spin_unlock(&mc.lock);
4761                         mc.moving_task = current;
4762                         mc.mm = mm;
4763
4764                         ret = mem_cgroup_precharge_mc(mm);
4765                         if (ret)
4766                                 mem_cgroup_clear_mc();
4767                         /* We call up_read() and mmput() in clear_mc(). */
4768                 } else
4769                         mmput(mm);
4770         }
4771         return ret;
4772 }
4773
4774 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4775                                 struct cgroup *cgroup,
4776                                 struct task_struct *p,
4777                                 bool threadgroup)
4778 {
4779         mem_cgroup_clear_mc();
4780 }
4781
4782 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4783                                 unsigned long addr, unsigned long end,
4784                                 struct mm_walk *walk)
4785 {
4786         int ret = 0;
4787         struct vm_area_struct *vma = walk->private;
4788         pte_t *pte;
4789         spinlock_t *ptl;
4790
4791 retry:
4792         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4793         for (; addr != end; addr += PAGE_SIZE) {
4794                 pte_t ptent = *(pte++);
4795                 union mc_target target;
4796                 int type;
4797                 struct page *page;
4798                 struct page_cgroup *pc;
4799                 swp_entry_t ent;
4800
4801                 if (!mc.precharge)
4802                         break;
4803
4804                 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4805                 switch (type) {
4806                 case MC_TARGET_PAGE:
4807                         page = target.page;
4808                         if (isolate_lru_page(page))
4809                                 goto put;
4810                         pc = lookup_page_cgroup(page);
4811                         if (!mem_cgroup_move_account(pc,
4812                                                 mc.from, mc.to, false)) {
4813                                 mc.precharge--;
4814                                 /* we uncharge from mc.from later. */
4815                                 mc.moved_charge++;
4816                         }
4817                         putback_lru_page(page);
4818 put:                    /* is_target_pte_for_mc() gets the page */
4819                         put_page(page);
4820                         break;
4821                 case MC_TARGET_SWAP:
4822                         ent = target.ent;
4823                         if (!mem_cgroup_move_swap_account(ent,
4824                                                 mc.from, mc.to, false)) {
4825                                 mc.precharge--;
4826                                 /* we fixup refcnts and charges later. */
4827                                 mc.moved_swap++;
4828                         }
4829                         break;
4830                 default:
4831                         break;
4832                 }
4833         }
4834         pte_unmap_unlock(pte - 1, ptl);
4835         cond_resched();
4836
4837         if (addr != end) {
4838                 /*
4839                  * We have consumed all precharges we got in can_attach().
4840                  * We try charge one by one, but don't do any additional
4841                  * charges to mc.to if we have failed in charge once in attach()
4842                  * phase.
4843                  */
4844                 ret = mem_cgroup_do_precharge(1);
4845                 if (!ret)
4846                         goto retry;
4847         }
4848
4849         return ret;
4850 }
4851
4852 static void mem_cgroup_move_charge(struct mm_struct *mm)
4853 {
4854         struct vm_area_struct *vma;
4855
4856         lru_add_drain_all();
4857         /* We've already held the mmap_sem */
4858         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4859                 int ret;
4860                 struct mm_walk mem_cgroup_move_charge_walk = {
4861                         .pmd_entry = mem_cgroup_move_charge_pte_range,
4862                         .mm = mm,
4863                         .private = vma,
4864                 };
4865                 if (is_vm_hugetlb_page(vma))
4866                         continue;
4867                 ret = walk_page_range(vma->vm_start, vma->vm_end,
4868                                                 &mem_cgroup_move_charge_walk);
4869                 if (ret)
4870                         /*
4871                          * means we have consumed all precharges and failed in
4872                          * doing additional charge. Just abandon here.
4873                          */
4874                         break;
4875         }
4876 }
4877
4878 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4879                                 struct cgroup *cont,
4880                                 struct cgroup *old_cont,
4881                                 struct task_struct *p,
4882                                 bool threadgroup)
4883 {
4884         if (!mc.mm)
4885                 /* no need to move charge */
4886                 return;
4887
4888         mem_cgroup_move_charge(mc.mm);
4889         mem_cgroup_clear_mc();
4890 }
4891 #else   /* !CONFIG_MMU */
4892 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4893                                 struct cgroup *cgroup,
4894                                 struct task_struct *p,
4895                                 bool threadgroup)
4896 {
4897         return 0;
4898 }
4899 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4900                                 struct cgroup *cgroup,
4901                                 struct task_struct *p,
4902                                 bool threadgroup)
4903 {
4904 }
4905 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4906                                 struct cgroup *cont,
4907                                 struct cgroup *old_cont,
4908                                 struct task_struct *p,
4909                                 bool threadgroup)
4910 {
4911 }
4912 #endif
4913
4914 struct cgroup_subsys mem_cgroup_subsys = {
4915         .name = "memory",
4916         .subsys_id = mem_cgroup_subsys_id,
4917         .create = mem_cgroup_create,
4918         .pre_destroy = mem_cgroup_pre_destroy,
4919         .destroy = mem_cgroup_destroy,
4920         .populate = mem_cgroup_populate,
4921         .can_attach = mem_cgroup_can_attach,
4922         .cancel_attach = mem_cgroup_cancel_attach,
4923         .attach = mem_cgroup_move_task,
4924         .early_init = 0,
4925         .use_id = 1,
4926 };
4927
4928 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4929 static int __init enable_swap_account(char *s)
4930 {
4931         /* consider enabled if no parameter or 1 is given */
4932         if (!s || !strcmp(s, "1"))
4933                 really_do_swap_account = 1;
4934         else if (!strcmp(s, "0"))
4935                 really_do_swap_account = 0;
4936         return 1;
4937 }
4938 __setup("swapaccount", enable_swap_account);
4939
4940 static int __init disable_swap_account(char *s)
4941 {
4942         enable_swap_account("0");
4943         return 1;
4944 }
4945 __setup("noswapaccount", disable_swap_account);
4946 #endif