7a22b41292115f78dc039ebf5e271090dc5a8f9d
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  */
23
24 #include <linux/res_counter.h>
25 #include <linux/memcontrol.h>
26 #include <linux/cgroup.h>
27 #include <linux/mm.h>
28 #include <linux/hugetlb.h>
29 #include <linux/pagemap.h>
30 #include <linux/smp.h>
31 #include <linux/page-flags.h>
32 #include <linux/backing-dev.h>
33 #include <linux/bit_spinlock.h>
34 #include <linux/rcupdate.h>
35 #include <linux/limits.h>
36 #include <linux/mutex.h>
37 #include <linux/rbtree.h>
38 #include <linux/slab.h>
39 #include <linux/swap.h>
40 #include <linux/swapops.h>
41 #include <linux/spinlock.h>
42 #include <linux/eventfd.h>
43 #include <linux/sort.h>
44 #include <linux/fs.h>
45 #include <linux/seq_file.h>
46 #include <linux/vmalloc.h>
47 #include <linux/mm_inline.h>
48 #include <linux/page_cgroup.h>
49 #include <linux/cpu.h>
50 #include <linux/oom.h>
51 #include "internal.h"
52
53 #include <asm/uaccess.h>
54
55 #include <trace/events/vmscan.h>
56
57 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
58 #define MEM_CGROUP_RECLAIM_RETRIES      5
59 struct mem_cgroup *root_mem_cgroup __read_mostly;
60
61 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
62 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
63 int do_swap_account __read_mostly;
64
65 /* for remember boot option*/
66 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP_ENABLED
67 static int really_do_swap_account __initdata = 1;
68 #else
69 static int really_do_swap_account __initdata = 0;
70 #endif
71
72 #else
73 #define do_swap_account         (0)
74 #endif
75
76 /*
77  * Per memcg event counter is incremented at every pagein/pageout. This counter
78  * is used for trigger some periodic events. This is straightforward and better
79  * than using jiffies etc. to handle periodic memcg event.
80  *
81  * These values will be used as !((event) & ((1 <<(thresh)) - 1))
82  */
83 #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
84 #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
85
86 /*
87  * Statistics for memory cgroup.
88  */
89 enum mem_cgroup_stat_index {
90         /*
91          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
92          */
93         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
94         MEM_CGROUP_STAT_RSS,       /* # of pages charged as anon rss */
95         MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
96         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
97         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
98         MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
99         MEM_CGROUP_STAT_DATA, /* end of data requires synchronization */
100         /* incremented at every  pagein/pageout */
101         MEM_CGROUP_EVENTS = MEM_CGROUP_STAT_DATA,
102         MEM_CGROUP_ON_MOVE,     /* someone is moving account between groups */
103
104         MEM_CGROUP_STAT_NSTATS,
105 };
106
107 struct mem_cgroup_stat_cpu {
108         s64 count[MEM_CGROUP_STAT_NSTATS];
109 };
110
111 /*
112  * per-zone information in memory controller.
113  */
114 struct mem_cgroup_per_zone {
115         /*
116          * spin_lock to protect the per cgroup LRU
117          */
118         struct list_head        lists[NR_LRU_LISTS];
119         unsigned long           count[NR_LRU_LISTS];
120
121         struct zone_reclaim_stat reclaim_stat;
122         struct rb_node          tree_node;      /* RB tree node */
123         unsigned long long      usage_in_excess;/* Set to the value by which */
124                                                 /* the soft limit is exceeded*/
125         bool                    on_tree;
126         struct mem_cgroup       *mem;           /* Back pointer, we cannot */
127                                                 /* use container_of        */
128 };
129 /* Macro for accessing counter */
130 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
131
132 struct mem_cgroup_per_node {
133         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
134 };
135
136 struct mem_cgroup_lru_info {
137         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
138 };
139
140 /*
141  * Cgroups above their limits are maintained in a RB-Tree, independent of
142  * their hierarchy representation
143  */
144
145 struct mem_cgroup_tree_per_zone {
146         struct rb_root rb_root;
147         spinlock_t lock;
148 };
149
150 struct mem_cgroup_tree_per_node {
151         struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
152 };
153
154 struct mem_cgroup_tree {
155         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
156 };
157
158 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
159
160 struct mem_cgroup_threshold {
161         struct eventfd_ctx *eventfd;
162         u64 threshold;
163 };
164
165 /* For threshold */
166 struct mem_cgroup_threshold_ary {
167         /* An array index points to threshold just below usage. */
168         int current_threshold;
169         /* Size of entries[] */
170         unsigned int size;
171         /* Array of thresholds */
172         struct mem_cgroup_threshold entries[0];
173 };
174
175 struct mem_cgroup_thresholds {
176         /* Primary thresholds array */
177         struct mem_cgroup_threshold_ary *primary;
178         /*
179          * Spare threshold array.
180          * This is needed to make mem_cgroup_unregister_event() "never fail".
181          * It must be able to store at least primary->size - 1 entries.
182          */
183         struct mem_cgroup_threshold_ary *spare;
184 };
185
186 /* for OOM */
187 struct mem_cgroup_eventfd_list {
188         struct list_head list;
189         struct eventfd_ctx *eventfd;
190 };
191
192 static void mem_cgroup_threshold(struct mem_cgroup *mem);
193 static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
194
195 /*
196  * The memory controller data structure. The memory controller controls both
197  * page cache and RSS per cgroup. We would eventually like to provide
198  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
199  * to help the administrator determine what knobs to tune.
200  *
201  * TODO: Add a water mark for the memory controller. Reclaim will begin when
202  * we hit the water mark. May be even add a low water mark, such that
203  * no reclaim occurs from a cgroup at it's low water mark, this is
204  * a feature that will be implemented much later in the future.
205  */
206 struct mem_cgroup {
207         struct cgroup_subsys_state css;
208         /*
209          * the counter to account for memory usage
210          */
211         struct res_counter res;
212         /*
213          * the counter to account for mem+swap usage.
214          */
215         struct res_counter memsw;
216         /*
217          * Per cgroup active and inactive list, similar to the
218          * per zone LRU lists.
219          */
220         struct mem_cgroup_lru_info info;
221
222         /*
223           protect against reclaim related member.
224         */
225         spinlock_t reclaim_param_lock;
226
227         /*
228          * While reclaiming in a hierarchy, we cache the last child we
229          * reclaimed from.
230          */
231         int last_scanned_child;
232         /*
233          * Should the accounting and control be hierarchical, per subtree?
234          */
235         bool use_hierarchy;
236         atomic_t        oom_lock;
237         atomic_t        refcnt;
238
239         unsigned int    swappiness;
240         /* OOM-Killer disable */
241         int             oom_kill_disable;
242
243         /* set when res.limit == memsw.limit */
244         bool            memsw_is_minimum;
245
246         /* protect arrays of thresholds */
247         struct mutex thresholds_lock;
248
249         /* thresholds for memory usage. RCU-protected */
250         struct mem_cgroup_thresholds thresholds;
251
252         /* thresholds for mem+swap usage. RCU-protected */
253         struct mem_cgroup_thresholds memsw_thresholds;
254
255         /* For oom notifier event fd */
256         struct list_head oom_notify;
257
258         /*
259          * Should we move charges of a task when a task is moved into this
260          * mem_cgroup ? And what type of charges should we move ?
261          */
262         unsigned long   move_charge_at_immigrate;
263         /*
264          * percpu counter.
265          */
266         struct mem_cgroup_stat_cpu *stat;
267         /*
268          * used when a cpu is offlined or other synchronizations
269          * See mem_cgroup_read_stat().
270          */
271         struct mem_cgroup_stat_cpu nocpu_base;
272         spinlock_t pcp_counter_lock;
273 };
274
275 /* Stuffs for move charges at task migration. */
276 /*
277  * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
278  * left-shifted bitmap of these types.
279  */
280 enum move_type {
281         MOVE_CHARGE_TYPE_ANON,  /* private anonymous page and swap of it */
282         MOVE_CHARGE_TYPE_FILE,  /* file page(including tmpfs) and swap of it */
283         NR_MOVE_TYPE,
284 };
285
286 /* "mc" and its members are protected by cgroup_mutex */
287 static struct move_charge_struct {
288         spinlock_t        lock; /* for from, to */
289         struct mem_cgroup *from;
290         struct mem_cgroup *to;
291         unsigned long precharge;
292         unsigned long moved_charge;
293         unsigned long moved_swap;
294         struct task_struct *moving_task;        /* a task moving charges */
295         struct mm_struct *mm;
296         wait_queue_head_t waitq;                /* a waitq for other context */
297 } mc = {
298         .lock = __SPIN_LOCK_UNLOCKED(mc.lock),
299         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
300 };
301
302 static bool move_anon(void)
303 {
304         return test_bit(MOVE_CHARGE_TYPE_ANON,
305                                         &mc.to->move_charge_at_immigrate);
306 }
307
308 static bool move_file(void)
309 {
310         return test_bit(MOVE_CHARGE_TYPE_FILE,
311                                         &mc.to->move_charge_at_immigrate);
312 }
313
314 /*
315  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
316  * limit reclaim to prevent infinite loops, if they ever occur.
317  */
318 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            (100)
319 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
320
321 enum charge_type {
322         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
323         MEM_CGROUP_CHARGE_TYPE_MAPPED,
324         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
325         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
326         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
327         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
328         NR_CHARGE_TYPE,
329 };
330
331 /* only for here (for easy reading.) */
332 #define PCGF_CACHE      (1UL << PCG_CACHE)
333 #define PCGF_USED       (1UL << PCG_USED)
334 #define PCGF_LOCK       (1UL << PCG_LOCK)
335 /* Not used, but added here for completeness */
336 #define PCGF_ACCT       (1UL << PCG_ACCT)
337
338 /* for encoding cft->private value on file */
339 #define _MEM                    (0)
340 #define _MEMSWAP                (1)
341 #define _OOM_TYPE               (2)
342 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
343 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
344 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
345 /* Used for OOM nofiier */
346 #define OOM_CONTROL             (0)
347
348 /*
349  * Reclaim flags for mem_cgroup_hierarchical_reclaim
350  */
351 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
352 #define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
353 #define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
354 #define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
355 #define MEM_CGROUP_RECLAIM_SOFT_BIT     0x2
356 #define MEM_CGROUP_RECLAIM_SOFT         (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
357
358 static void mem_cgroup_get(struct mem_cgroup *mem);
359 static void mem_cgroup_put(struct mem_cgroup *mem);
360 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
361 static void drain_all_stock_async(void);
362
363 static struct mem_cgroup_per_zone *
364 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
365 {
366         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
367 }
368
369 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
370 {
371         return &mem->css;
372 }
373
374 static struct mem_cgroup_per_zone *
375 page_cgroup_zoneinfo(struct page_cgroup *pc)
376 {
377         struct mem_cgroup *mem = pc->mem_cgroup;
378         int nid = page_cgroup_nid(pc);
379         int zid = page_cgroup_zid(pc);
380
381         if (!mem)
382                 return NULL;
383
384         return mem_cgroup_zoneinfo(mem, nid, zid);
385 }
386
387 static struct mem_cgroup_tree_per_zone *
388 soft_limit_tree_node_zone(int nid, int zid)
389 {
390         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
391 }
392
393 static struct mem_cgroup_tree_per_zone *
394 soft_limit_tree_from_page(struct page *page)
395 {
396         int nid = page_to_nid(page);
397         int zid = page_zonenum(page);
398
399         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
400 }
401
402 static void
403 __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
404                                 struct mem_cgroup_per_zone *mz,
405                                 struct mem_cgroup_tree_per_zone *mctz,
406                                 unsigned long long new_usage_in_excess)
407 {
408         struct rb_node **p = &mctz->rb_root.rb_node;
409         struct rb_node *parent = NULL;
410         struct mem_cgroup_per_zone *mz_node;
411
412         if (mz->on_tree)
413                 return;
414
415         mz->usage_in_excess = new_usage_in_excess;
416         if (!mz->usage_in_excess)
417                 return;
418         while (*p) {
419                 parent = *p;
420                 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
421                                         tree_node);
422                 if (mz->usage_in_excess < mz_node->usage_in_excess)
423                         p = &(*p)->rb_left;
424                 /*
425                  * We can't avoid mem cgroups that are over their soft
426                  * limit by the same amount
427                  */
428                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
429                         p = &(*p)->rb_right;
430         }
431         rb_link_node(&mz->tree_node, parent, p);
432         rb_insert_color(&mz->tree_node, &mctz->rb_root);
433         mz->on_tree = true;
434 }
435
436 static void
437 __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
438                                 struct mem_cgroup_per_zone *mz,
439                                 struct mem_cgroup_tree_per_zone *mctz)
440 {
441         if (!mz->on_tree)
442                 return;
443         rb_erase(&mz->tree_node, &mctz->rb_root);
444         mz->on_tree = false;
445 }
446
447 static void
448 mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
449                                 struct mem_cgroup_per_zone *mz,
450                                 struct mem_cgroup_tree_per_zone *mctz)
451 {
452         spin_lock(&mctz->lock);
453         __mem_cgroup_remove_exceeded(mem, mz, mctz);
454         spin_unlock(&mctz->lock);
455 }
456
457
458 static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
459 {
460         unsigned long long excess;
461         struct mem_cgroup_per_zone *mz;
462         struct mem_cgroup_tree_per_zone *mctz;
463         int nid = page_to_nid(page);
464         int zid = page_zonenum(page);
465         mctz = soft_limit_tree_from_page(page);
466
467         /*
468          * Necessary to update all ancestors when hierarchy is used.
469          * because their event counter is not touched.
470          */
471         for (; mem; mem = parent_mem_cgroup(mem)) {
472                 mz = mem_cgroup_zoneinfo(mem, nid, zid);
473                 excess = res_counter_soft_limit_excess(&mem->res);
474                 /*
475                  * We have to update the tree if mz is on RB-tree or
476                  * mem is over its softlimit.
477                  */
478                 if (excess || mz->on_tree) {
479                         spin_lock(&mctz->lock);
480                         /* if on-tree, remove it */
481                         if (mz->on_tree)
482                                 __mem_cgroup_remove_exceeded(mem, mz, mctz);
483                         /*
484                          * Insert again. mz->usage_in_excess will be updated.
485                          * If excess is 0, no tree ops.
486                          */
487                         __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
488                         spin_unlock(&mctz->lock);
489                 }
490         }
491 }
492
493 static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
494 {
495         int node, zone;
496         struct mem_cgroup_per_zone *mz;
497         struct mem_cgroup_tree_per_zone *mctz;
498
499         for_each_node_state(node, N_POSSIBLE) {
500                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
501                         mz = mem_cgroup_zoneinfo(mem, node, zone);
502                         mctz = soft_limit_tree_node_zone(node, zone);
503                         mem_cgroup_remove_exceeded(mem, mz, mctz);
504                 }
505         }
506 }
507
508 static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
509 {
510         return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
511 }
512
513 static struct mem_cgroup_per_zone *
514 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
515 {
516         struct rb_node *rightmost = NULL;
517         struct mem_cgroup_per_zone *mz;
518
519 retry:
520         mz = NULL;
521         rightmost = rb_last(&mctz->rb_root);
522         if (!rightmost)
523                 goto done;              /* Nothing to reclaim from */
524
525         mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
526         /*
527          * Remove the node now but someone else can add it back,
528          * we will to add it back at the end of reclaim to its correct
529          * position in the tree.
530          */
531         __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
532         if (!res_counter_soft_limit_excess(&mz->mem->res) ||
533                 !css_tryget(&mz->mem->css))
534                 goto retry;
535 done:
536         return mz;
537 }
538
539 static struct mem_cgroup_per_zone *
540 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
541 {
542         struct mem_cgroup_per_zone *mz;
543
544         spin_lock(&mctz->lock);
545         mz = __mem_cgroup_largest_soft_limit_node(mctz);
546         spin_unlock(&mctz->lock);
547         return mz;
548 }
549
550 /*
551  * Implementation Note: reading percpu statistics for memcg.
552  *
553  * Both of vmstat[] and percpu_counter has threshold and do periodic
554  * synchronization to implement "quick" read. There are trade-off between
555  * reading cost and precision of value. Then, we may have a chance to implement
556  * a periodic synchronizion of counter in memcg's counter.
557  *
558  * But this _read() function is used for user interface now. The user accounts
559  * memory usage by memory cgroup and he _always_ requires exact value because
560  * he accounts memory. Even if we provide quick-and-fuzzy read, we always
561  * have to visit all online cpus and make sum. So, for now, unnecessary
562  * synchronization is not implemented. (just implemented for cpu hotplug)
563  *
564  * If there are kernel internal actions which can make use of some not-exact
565  * value, and reading all cpu value can be performance bottleneck in some
566  * common workload, threashold and synchonization as vmstat[] should be
567  * implemented.
568  */
569 static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
570                 enum mem_cgroup_stat_index idx)
571 {
572         int cpu;
573         s64 val = 0;
574
575         get_online_cpus();
576         for_each_online_cpu(cpu)
577                 val += per_cpu(mem->stat->count[idx], cpu);
578 #ifdef CONFIG_HOTPLUG_CPU
579         spin_lock(&mem->pcp_counter_lock);
580         val += mem->nocpu_base.count[idx];
581         spin_unlock(&mem->pcp_counter_lock);
582 #endif
583         put_online_cpus();
584         return val;
585 }
586
587 static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
588 {
589         s64 ret;
590
591         ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
592         ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
593         return ret;
594 }
595
596 static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
597                                          bool charge)
598 {
599         int val = (charge) ? 1 : -1;
600         this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
601 }
602
603 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
604                                          struct page_cgroup *pc,
605                                          bool charge)
606 {
607         int val = (charge) ? 1 : -1;
608
609         preempt_disable();
610
611         if (PageCgroupCache(pc))
612                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
613         else
614                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
615
616         if (charge)
617                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
618         else
619                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
620         __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
621
622         preempt_enable();
623 }
624
625 static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
626                                         enum lru_list idx)
627 {
628         int nid, zid;
629         struct mem_cgroup_per_zone *mz;
630         u64 total = 0;
631
632         for_each_online_node(nid)
633                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
634                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
635                         total += MEM_CGROUP_ZSTAT(mz, idx);
636                 }
637         return total;
638 }
639
640 static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
641 {
642         s64 val;
643
644         val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
645
646         return !(val & ((1 << event_mask_shift) - 1));
647 }
648
649 /*
650  * Check events in order.
651  *
652  */
653 static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
654 {
655         /* threshold event is triggered in finer grain than soft limit */
656         if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
657                 mem_cgroup_threshold(mem);
658                 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
659                         mem_cgroup_update_tree(mem, page);
660         }
661 }
662
663 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
664 {
665         return container_of(cgroup_subsys_state(cont,
666                                 mem_cgroup_subsys_id), struct mem_cgroup,
667                                 css);
668 }
669
670 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
671 {
672         /*
673          * mm_update_next_owner() may clear mm->owner to NULL
674          * if it races with swapoff, page migration, etc.
675          * So this can be called with p == NULL.
676          */
677         if (unlikely(!p))
678                 return NULL;
679
680         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
681                                 struct mem_cgroup, css);
682 }
683
684 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
685 {
686         struct mem_cgroup *mem = NULL;
687
688         if (!mm)
689                 return NULL;
690         /*
691          * Because we have no locks, mm->owner's may be being moved to other
692          * cgroup. We use css_tryget() here even if this looks
693          * pessimistic (rather than adding locks here).
694          */
695         rcu_read_lock();
696         do {
697                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
698                 if (unlikely(!mem))
699                         break;
700         } while (!css_tryget(&mem->css));
701         rcu_read_unlock();
702         return mem;
703 }
704
705 /* The caller has to guarantee "mem" exists before calling this */
706 static struct mem_cgroup *mem_cgroup_start_loop(struct mem_cgroup *mem)
707 {
708         struct cgroup_subsys_state *css;
709         int found;
710
711         if (!mem) /* ROOT cgroup has the smallest ID */
712                 return root_mem_cgroup; /*css_put/get against root is ignored*/
713         if (!mem->use_hierarchy) {
714                 if (css_tryget(&mem->css))
715                         return mem;
716                 return NULL;
717         }
718         rcu_read_lock();
719         /*
720          * searching a memory cgroup which has the smallest ID under given
721          * ROOT cgroup. (ID >= 1)
722          */
723         css = css_get_next(&mem_cgroup_subsys, 1, &mem->css, &found);
724         if (css && css_tryget(css))
725                 mem = container_of(css, struct mem_cgroup, css);
726         else
727                 mem = NULL;
728         rcu_read_unlock();
729         return mem;
730 }
731
732 static struct mem_cgroup *mem_cgroup_get_next(struct mem_cgroup *iter,
733                                         struct mem_cgroup *root,
734                                         bool cond)
735 {
736         int nextid = css_id(&iter->css) + 1;
737         int found;
738         int hierarchy_used;
739         struct cgroup_subsys_state *css;
740
741         hierarchy_used = iter->use_hierarchy;
742
743         css_put(&iter->css);
744         /* If no ROOT, walk all, ignore hierarchy */
745         if (!cond || (root && !hierarchy_used))
746                 return NULL;
747
748         if (!root)
749                 root = root_mem_cgroup;
750
751         do {
752                 iter = NULL;
753                 rcu_read_lock();
754
755                 css = css_get_next(&mem_cgroup_subsys, nextid,
756                                 &root->css, &found);
757                 if (css && css_tryget(css))
758                         iter = container_of(css, struct mem_cgroup, css);
759                 rcu_read_unlock();
760                 /* If css is NULL, no more cgroups will be found */
761                 nextid = found + 1;
762         } while (css && !iter);
763
764         return iter;
765 }
766 /*
767  * for_eacn_mem_cgroup_tree() for visiting all cgroup under tree. Please
768  * be careful that "break" loop is not allowed. We have reference count.
769  * Instead of that modify "cond" to be false and "continue" to exit the loop.
770  */
771 #define for_each_mem_cgroup_tree_cond(iter, root, cond) \
772         for (iter = mem_cgroup_start_loop(root);\
773              iter != NULL;\
774              iter = mem_cgroup_get_next(iter, root, cond))
775
776 #define for_each_mem_cgroup_tree(iter, root) \
777         for_each_mem_cgroup_tree_cond(iter, root, true)
778
779 #define for_each_mem_cgroup_all(iter) \
780         for_each_mem_cgroup_tree_cond(iter, NULL, true)
781
782
783 static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
784 {
785         return (mem == root_mem_cgroup);
786 }
787
788 /*
789  * Following LRU functions are allowed to be used without PCG_LOCK.
790  * Operations are called by routine of global LRU independently from memcg.
791  * What we have to take care of here is validness of pc->mem_cgroup.
792  *
793  * Changes to pc->mem_cgroup happens when
794  * 1. charge
795  * 2. moving account
796  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
797  * It is added to LRU before charge.
798  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
799  * When moving account, the page is not on LRU. It's isolated.
800  */
801
802 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
803 {
804         struct page_cgroup *pc;
805         struct mem_cgroup_per_zone *mz;
806
807         if (mem_cgroup_disabled())
808                 return;
809         pc = lookup_page_cgroup(page);
810         /* can happen while we handle swapcache. */
811         if (!TestClearPageCgroupAcctLRU(pc))
812                 return;
813         VM_BUG_ON(!pc->mem_cgroup);
814         /*
815          * We don't check PCG_USED bit. It's cleared when the "page" is finally
816          * removed from global LRU.
817          */
818         mz = page_cgroup_zoneinfo(pc);
819         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
820         if (mem_cgroup_is_root(pc->mem_cgroup))
821                 return;
822         VM_BUG_ON(list_empty(&pc->lru));
823         list_del_init(&pc->lru);
824         return;
825 }
826
827 void mem_cgroup_del_lru(struct page *page)
828 {
829         mem_cgroup_del_lru_list(page, page_lru(page));
830 }
831
832 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
833 {
834         struct mem_cgroup_per_zone *mz;
835         struct page_cgroup *pc;
836
837         if (mem_cgroup_disabled())
838                 return;
839
840         pc = lookup_page_cgroup(page);
841         /*
842          * Used bit is set without atomic ops but after smp_wmb().
843          * For making pc->mem_cgroup visible, insert smp_rmb() here.
844          */
845         smp_rmb();
846         /* unused or root page is not rotated. */
847         if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
848                 return;
849         mz = page_cgroup_zoneinfo(pc);
850         list_move(&pc->lru, &mz->lists[lru]);
851 }
852
853 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
854 {
855         struct page_cgroup *pc;
856         struct mem_cgroup_per_zone *mz;
857
858         if (mem_cgroup_disabled())
859                 return;
860         pc = lookup_page_cgroup(page);
861         VM_BUG_ON(PageCgroupAcctLRU(pc));
862         /*
863          * Used bit is set without atomic ops but after smp_wmb().
864          * For making pc->mem_cgroup visible, insert smp_rmb() here.
865          */
866         smp_rmb();
867         if (!PageCgroupUsed(pc))
868                 return;
869
870         mz = page_cgroup_zoneinfo(pc);
871         MEM_CGROUP_ZSTAT(mz, lru) += 1;
872         SetPageCgroupAcctLRU(pc);
873         if (mem_cgroup_is_root(pc->mem_cgroup))
874                 return;
875         list_add(&pc->lru, &mz->lists[lru]);
876 }
877
878 /*
879  * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
880  * lru because the page may.be reused after it's fully uncharged (because of
881  * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
882  * it again. This function is only used to charge SwapCache. It's done under
883  * lock_page and expected that zone->lru_lock is never held.
884  */
885 static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
886 {
887         unsigned long flags;
888         struct zone *zone = page_zone(page);
889         struct page_cgroup *pc = lookup_page_cgroup(page);
890
891         spin_lock_irqsave(&zone->lru_lock, flags);
892         /*
893          * Forget old LRU when this page_cgroup is *not* used. This Used bit
894          * is guarded by lock_page() because the page is SwapCache.
895          */
896         if (!PageCgroupUsed(pc))
897                 mem_cgroup_del_lru_list(page, page_lru(page));
898         spin_unlock_irqrestore(&zone->lru_lock, flags);
899 }
900
901 static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
902 {
903         unsigned long flags;
904         struct zone *zone = page_zone(page);
905         struct page_cgroup *pc = lookup_page_cgroup(page);
906
907         spin_lock_irqsave(&zone->lru_lock, flags);
908         /* link when the page is linked to LRU but page_cgroup isn't */
909         if (PageLRU(page) && !PageCgroupAcctLRU(pc))
910                 mem_cgroup_add_lru_list(page, page_lru(page));
911         spin_unlock_irqrestore(&zone->lru_lock, flags);
912 }
913
914
915 void mem_cgroup_move_lists(struct page *page,
916                            enum lru_list from, enum lru_list to)
917 {
918         if (mem_cgroup_disabled())
919                 return;
920         mem_cgroup_del_lru_list(page, from);
921         mem_cgroup_add_lru_list(page, to);
922 }
923
924 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
925 {
926         int ret;
927         struct mem_cgroup *curr = NULL;
928         struct task_struct *p;
929
930         p = find_lock_task_mm(task);
931         if (!p)
932                 return 0;
933         curr = try_get_mem_cgroup_from_mm(p->mm);
934         task_unlock(p);
935         if (!curr)
936                 return 0;
937         /*
938          * We should check use_hierarchy of "mem" not "curr". Because checking
939          * use_hierarchy of "curr" here make this function true if hierarchy is
940          * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
941          * hierarchy(even if use_hierarchy is disabled in "mem").
942          */
943         if (mem->use_hierarchy)
944                 ret = css_is_ancestor(&curr->css, &mem->css);
945         else
946                 ret = (curr == mem);
947         css_put(&curr->css);
948         return ret;
949 }
950
951 static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
952 {
953         unsigned long active;
954         unsigned long inactive;
955         unsigned long gb;
956         unsigned long inactive_ratio;
957
958         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
959         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
960
961         gb = (inactive + active) >> (30 - PAGE_SHIFT);
962         if (gb)
963                 inactive_ratio = int_sqrt(10 * gb);
964         else
965                 inactive_ratio = 1;
966
967         if (present_pages) {
968                 present_pages[0] = inactive;
969                 present_pages[1] = active;
970         }
971
972         return inactive_ratio;
973 }
974
975 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
976 {
977         unsigned long active;
978         unsigned long inactive;
979         unsigned long present_pages[2];
980         unsigned long inactive_ratio;
981
982         inactive_ratio = calc_inactive_ratio(memcg, present_pages);
983
984         inactive = present_pages[0];
985         active = present_pages[1];
986
987         if (inactive * inactive_ratio < active)
988                 return 1;
989
990         return 0;
991 }
992
993 int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
994 {
995         unsigned long active;
996         unsigned long inactive;
997
998         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
999         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
1000
1001         return (active > inactive);
1002 }
1003
1004 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
1005                                        struct zone *zone,
1006                                        enum lru_list lru)
1007 {
1008         int nid = zone_to_nid(zone);
1009         int zid = zone_idx(zone);
1010         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1011
1012         return MEM_CGROUP_ZSTAT(mz, lru);
1013 }
1014
1015 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
1016                                                       struct zone *zone)
1017 {
1018         int nid = zone_to_nid(zone);
1019         int zid = zone_idx(zone);
1020         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
1021
1022         return &mz->reclaim_stat;
1023 }
1024
1025 struct zone_reclaim_stat *
1026 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
1027 {
1028         struct page_cgroup *pc;
1029         struct mem_cgroup_per_zone *mz;
1030
1031         if (mem_cgroup_disabled())
1032                 return NULL;
1033
1034         pc = lookup_page_cgroup(page);
1035         /*
1036          * Used bit is set without atomic ops but after smp_wmb().
1037          * For making pc->mem_cgroup visible, insert smp_rmb() here.
1038          */
1039         smp_rmb();
1040         if (!PageCgroupUsed(pc))
1041                 return NULL;
1042
1043         mz = page_cgroup_zoneinfo(pc);
1044         if (!mz)
1045                 return NULL;
1046
1047         return &mz->reclaim_stat;
1048 }
1049
1050 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
1051                                         struct list_head *dst,
1052                                         unsigned long *scanned, int order,
1053                                         int mode, struct zone *z,
1054                                         struct mem_cgroup *mem_cont,
1055                                         int active, int file)
1056 {
1057         unsigned long nr_taken = 0;
1058         struct page *page;
1059         unsigned long scan;
1060         LIST_HEAD(pc_list);
1061         struct list_head *src;
1062         struct page_cgroup *pc, *tmp;
1063         int nid = zone_to_nid(z);
1064         int zid = zone_idx(z);
1065         struct mem_cgroup_per_zone *mz;
1066         int lru = LRU_FILE * file + active;
1067         int ret;
1068
1069         BUG_ON(!mem_cont);
1070         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
1071         src = &mz->lists[lru];
1072
1073         scan = 0;
1074         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
1075                 if (scan >= nr_to_scan)
1076                         break;
1077
1078                 page = pc->page;
1079                 if (unlikely(!PageCgroupUsed(pc)))
1080                         continue;
1081                 if (unlikely(!PageLRU(page)))
1082                         continue;
1083
1084                 scan++;
1085                 ret = __isolate_lru_page(page, mode, file);
1086                 switch (ret) {
1087                 case 0:
1088                         list_move(&page->lru, dst);
1089                         mem_cgroup_del_lru(page);
1090                         nr_taken++;
1091                         break;
1092                 case -EBUSY:
1093                         /* we don't affect global LRU but rotate in our LRU */
1094                         mem_cgroup_rotate_lru_list(page, page_lru(page));
1095                         break;
1096                 default:
1097                         break;
1098                 }
1099         }
1100
1101         *scanned = scan;
1102
1103         trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1104                                       0, 0, 0, mode);
1105
1106         return nr_taken;
1107 }
1108
1109 #define mem_cgroup_from_res_counter(counter, member)    \
1110         container_of(counter, struct mem_cgroup, member)
1111
1112 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1113 {
1114         if (do_swap_account) {
1115                 if (res_counter_check_under_limit(&mem->res) &&
1116                         res_counter_check_under_limit(&mem->memsw))
1117                         return true;
1118         } else
1119                 if (res_counter_check_under_limit(&mem->res))
1120                         return true;
1121         return false;
1122 }
1123
1124 static unsigned int get_swappiness(struct mem_cgroup *memcg)
1125 {
1126         struct cgroup *cgrp = memcg->css.cgroup;
1127         unsigned int swappiness;
1128
1129         /* root ? */
1130         if (cgrp->parent == NULL)
1131                 return vm_swappiness;
1132
1133         spin_lock(&memcg->reclaim_param_lock);
1134         swappiness = memcg->swappiness;
1135         spin_unlock(&memcg->reclaim_param_lock);
1136
1137         return swappiness;
1138 }
1139
1140 static void mem_cgroup_start_move(struct mem_cgroup *mem)
1141 {
1142         int cpu;
1143
1144         get_online_cpus();
1145         spin_lock(&mem->pcp_counter_lock);
1146         for_each_online_cpu(cpu)
1147                 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) += 1;
1148         mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] += 1;
1149         spin_unlock(&mem->pcp_counter_lock);
1150         put_online_cpus();
1151
1152         synchronize_rcu();
1153 }
1154
1155 static void mem_cgroup_end_move(struct mem_cgroup *mem)
1156 {
1157         int cpu;
1158
1159         if (!mem)
1160                 return;
1161         get_online_cpus();
1162         spin_lock(&mem->pcp_counter_lock);
1163         for_each_online_cpu(cpu)
1164                 per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) -= 1;
1165         mem->nocpu_base.count[MEM_CGROUP_ON_MOVE] -= 1;
1166         spin_unlock(&mem->pcp_counter_lock);
1167         put_online_cpus();
1168 }
1169 /*
1170  * 2 routines for checking "mem" is under move_account() or not.
1171  *
1172  * mem_cgroup_stealed() - checking a cgroup is mc.from or not. This is used
1173  *                        for avoiding race in accounting. If true,
1174  *                        pc->mem_cgroup may be overwritten.
1175  *
1176  * mem_cgroup_under_move() - checking a cgroup is mc.from or mc.to or
1177  *                        under hierarchy of moving cgroups. This is for
1178  *                        waiting at hith-memory prressure caused by "move".
1179  */
1180
1181 static bool mem_cgroup_stealed(struct mem_cgroup *mem)
1182 {
1183         VM_BUG_ON(!rcu_read_lock_held());
1184         return this_cpu_read(mem->stat->count[MEM_CGROUP_ON_MOVE]) > 0;
1185 }
1186
1187 static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1188 {
1189         struct mem_cgroup *from;
1190         struct mem_cgroup *to;
1191         bool ret = false;
1192         /*
1193          * Unlike task_move routines, we access mc.to, mc.from not under
1194          * mutual exclusion by cgroup_mutex. Here, we take spinlock instead.
1195          */
1196         spin_lock(&mc.lock);
1197         from = mc.from;
1198         to = mc.to;
1199         if (!from)
1200                 goto unlock;
1201         if (from == mem || to == mem
1202             || (mem->use_hierarchy && css_is_ancestor(&from->css, &mem->css))
1203             || (mem->use_hierarchy && css_is_ancestor(&to->css, &mem->css)))
1204                 ret = true;
1205 unlock:
1206         spin_unlock(&mc.lock);
1207         return ret;
1208 }
1209
1210 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1211 {
1212         if (mc.moving_task && current != mc.moving_task) {
1213                 if (mem_cgroup_under_move(mem)) {
1214                         DEFINE_WAIT(wait);
1215                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1216                         /* moving charge context might have finished. */
1217                         if (mc.moving_task)
1218                                 schedule();
1219                         finish_wait(&mc.waitq, &wait);
1220                         return true;
1221                 }
1222         }
1223         return false;
1224 }
1225
1226 /**
1227  * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1228  * @memcg: The memory cgroup that went over limit
1229  * @p: Task that is going to be killed
1230  *
1231  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1232  * enabled
1233  */
1234 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1235 {
1236         struct cgroup *task_cgrp;
1237         struct cgroup *mem_cgrp;
1238         /*
1239          * Need a buffer in BSS, can't rely on allocations. The code relies
1240          * on the assumption that OOM is serialized for memory controller.
1241          * If this assumption is broken, revisit this code.
1242          */
1243         static char memcg_name[PATH_MAX];
1244         int ret;
1245
1246         if (!memcg || !p)
1247                 return;
1248
1249
1250         rcu_read_lock();
1251
1252         mem_cgrp = memcg->css.cgroup;
1253         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1254
1255         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1256         if (ret < 0) {
1257                 /*
1258                  * Unfortunately, we are unable to convert to a useful name
1259                  * But we'll still print out the usage information
1260                  */
1261                 rcu_read_unlock();
1262                 goto done;
1263         }
1264         rcu_read_unlock();
1265
1266         printk(KERN_INFO "Task in %s killed", memcg_name);
1267
1268         rcu_read_lock();
1269         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1270         if (ret < 0) {
1271                 rcu_read_unlock();
1272                 goto done;
1273         }
1274         rcu_read_unlock();
1275
1276         /*
1277          * Continues from above, so we don't need an KERN_ level
1278          */
1279         printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1280 done:
1281
1282         printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1283                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1284                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1285                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1286         printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1287                 "failcnt %llu\n",
1288                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1289                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1290                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1291 }
1292
1293 /*
1294  * This function returns the number of memcg under hierarchy tree. Returns
1295  * 1(self count) if no children.
1296  */
1297 static int mem_cgroup_count_children(struct mem_cgroup *mem)
1298 {
1299         int num = 0;
1300         struct mem_cgroup *iter;
1301
1302         for_each_mem_cgroup_tree(iter, mem)
1303                 num++;
1304         return num;
1305 }
1306
1307 /*
1308  * Return the memory (and swap, if configured) limit for a memcg.
1309  */
1310 u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1311 {
1312         u64 limit;
1313         u64 memsw;
1314
1315         limit = res_counter_read_u64(&memcg->res, RES_LIMIT) +
1316                         total_swap_pages;
1317         memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1318         /*
1319          * If memsw is finite and limits the amount of swap space available
1320          * to this memcg, return that limit.
1321          */
1322         return min(limit, memsw);
1323 }
1324
1325 /*
1326  * Visit the first child (need not be the first child as per the ordering
1327  * of the cgroup list, since we track last_scanned_child) of @mem and use
1328  * that to reclaim free pages from.
1329  */
1330 static struct mem_cgroup *
1331 mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1332 {
1333         struct mem_cgroup *ret = NULL;
1334         struct cgroup_subsys_state *css;
1335         int nextid, found;
1336
1337         if (!root_mem->use_hierarchy) {
1338                 css_get(&root_mem->css);
1339                 ret = root_mem;
1340         }
1341
1342         while (!ret) {
1343                 rcu_read_lock();
1344                 nextid = root_mem->last_scanned_child + 1;
1345                 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1346                                    &found);
1347                 if (css && css_tryget(css))
1348                         ret = container_of(css, struct mem_cgroup, css);
1349
1350                 rcu_read_unlock();
1351                 /* Updates scanning parameter */
1352                 spin_lock(&root_mem->reclaim_param_lock);
1353                 if (!css) {
1354                         /* this means start scan from ID:1 */
1355                         root_mem->last_scanned_child = 0;
1356                 } else
1357                         root_mem->last_scanned_child = found;
1358                 spin_unlock(&root_mem->reclaim_param_lock);
1359         }
1360
1361         return ret;
1362 }
1363
1364 /*
1365  * Scan the hierarchy if needed to reclaim memory. We remember the last child
1366  * we reclaimed from, so that we don't end up penalizing one child extensively
1367  * based on its position in the children list.
1368  *
1369  * root_mem is the original ancestor that we've been reclaim from.
1370  *
1371  * We give up and return to the caller when we visit root_mem twice.
1372  * (other groups can be removed while we're walking....)
1373  *
1374  * If shrink==true, for avoiding to free too much, this returns immedieately.
1375  */
1376 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
1377                                                 struct zone *zone,
1378                                                 gfp_t gfp_mask,
1379                                                 unsigned long reclaim_options)
1380 {
1381         struct mem_cgroup *victim;
1382         int ret, total = 0;
1383         int loop = 0;
1384         bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1385         bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1386         bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1387         unsigned long excess = mem_cgroup_get_excess(root_mem);
1388
1389         /* If memsw_is_minimum==1, swap-out is of-no-use. */
1390         if (root_mem->memsw_is_minimum)
1391                 noswap = true;
1392
1393         while (1) {
1394                 victim = mem_cgroup_select_victim(root_mem);
1395                 if (victim == root_mem) {
1396                         loop++;
1397                         if (loop >= 1)
1398                                 drain_all_stock_async();
1399                         if (loop >= 2) {
1400                                 /*
1401                                  * If we have not been able to reclaim
1402                                  * anything, it might because there are
1403                                  * no reclaimable pages under this hierarchy
1404                                  */
1405                                 if (!check_soft || !total) {
1406                                         css_put(&victim->css);
1407                                         break;
1408                                 }
1409                                 /*
1410                                  * We want to do more targetted reclaim.
1411                                  * excess >> 2 is not to excessive so as to
1412                                  * reclaim too much, nor too less that we keep
1413                                  * coming back to reclaim from this cgroup
1414                                  */
1415                                 if (total >= (excess >> 2) ||
1416                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1417                                         css_put(&victim->css);
1418                                         break;
1419                                 }
1420                         }
1421                 }
1422                 if (!mem_cgroup_local_usage(victim)) {
1423                         /* this cgroup's local usage == 0 */
1424                         css_put(&victim->css);
1425                         continue;
1426                 }
1427                 /* we use swappiness of local cgroup */
1428                 if (check_soft)
1429                         ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
1430                                 noswap, get_swappiness(victim), zone);
1431                 else
1432                         ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1433                                                 noswap, get_swappiness(victim));
1434                 css_put(&victim->css);
1435                 /*
1436                  * At shrinking usage, we can't check we should stop here or
1437                  * reclaim more. It's depends on callers. last_scanned_child
1438                  * will work enough for keeping fairness under tree.
1439                  */
1440                 if (shrink)
1441                         return ret;
1442                 total += ret;
1443                 if (check_soft) {
1444                         if (res_counter_check_under_soft_limit(&root_mem->res))
1445                                 return total;
1446                 } else if (mem_cgroup_check_under_limit(root_mem))
1447                         return 1 + total;
1448         }
1449         return total;
1450 }
1451
1452 /*
1453  * Check OOM-Killer is already running under our hierarchy.
1454  * If someone is running, return false.
1455  */
1456 static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1457 {
1458         int x, lock_count = 0;
1459         struct mem_cgroup *iter;
1460
1461         for_each_mem_cgroup_tree(iter, mem) {
1462                 x = atomic_inc_return(&iter->oom_lock);
1463                 lock_count = max(x, lock_count);
1464         }
1465
1466         if (lock_count == 1)
1467                 return true;
1468         return false;
1469 }
1470
1471 static int mem_cgroup_oom_unlock(struct mem_cgroup *mem)
1472 {
1473         struct mem_cgroup *iter;
1474
1475         /*
1476          * When a new child is created while the hierarchy is under oom,
1477          * mem_cgroup_oom_lock() may not be called. We have to use
1478          * atomic_add_unless() here.
1479          */
1480         for_each_mem_cgroup_tree(iter, mem)
1481                 atomic_add_unless(&iter->oom_lock, -1, 0);
1482         return 0;
1483 }
1484
1485
1486 static DEFINE_MUTEX(memcg_oom_mutex);
1487 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1488
1489 struct oom_wait_info {
1490         struct mem_cgroup *mem;
1491         wait_queue_t    wait;
1492 };
1493
1494 static int memcg_oom_wake_function(wait_queue_t *wait,
1495         unsigned mode, int sync, void *arg)
1496 {
1497         struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1498         struct oom_wait_info *oom_wait_info;
1499
1500         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1501
1502         if (oom_wait_info->mem == wake_mem)
1503                 goto wakeup;
1504         /* if no hierarchy, no match */
1505         if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1506                 return 0;
1507         /*
1508          * Both of oom_wait_info->mem and wake_mem are stable under us.
1509          * Then we can use css_is_ancestor without taking care of RCU.
1510          */
1511         if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1512             !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1513                 return 0;
1514
1515 wakeup:
1516         return autoremove_wake_function(wait, mode, sync, arg);
1517 }
1518
1519 static void memcg_wakeup_oom(struct mem_cgroup *mem)
1520 {
1521         /* for filtering, pass "mem" as argument. */
1522         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1523 }
1524
1525 static void memcg_oom_recover(struct mem_cgroup *mem)
1526 {
1527         if (mem && atomic_read(&mem->oom_lock))
1528                 memcg_wakeup_oom(mem);
1529 }
1530
1531 /*
1532  * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1533  */
1534 bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1535 {
1536         struct oom_wait_info owait;
1537         bool locked, need_to_kill;
1538
1539         owait.mem = mem;
1540         owait.wait.flags = 0;
1541         owait.wait.func = memcg_oom_wake_function;
1542         owait.wait.private = current;
1543         INIT_LIST_HEAD(&owait.wait.task_list);
1544         need_to_kill = true;
1545         /* At first, try to OOM lock hierarchy under mem.*/
1546         mutex_lock(&memcg_oom_mutex);
1547         locked = mem_cgroup_oom_lock(mem);
1548         /*
1549          * Even if signal_pending(), we can't quit charge() loop without
1550          * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1551          * under OOM is always welcomed, use TASK_KILLABLE here.
1552          */
1553         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1554         if (!locked || mem->oom_kill_disable)
1555                 need_to_kill = false;
1556         if (locked)
1557                 mem_cgroup_oom_notify(mem);
1558         mutex_unlock(&memcg_oom_mutex);
1559
1560         if (need_to_kill) {
1561                 finish_wait(&memcg_oom_waitq, &owait.wait);
1562                 mem_cgroup_out_of_memory(mem, mask);
1563         } else {
1564                 schedule();
1565                 finish_wait(&memcg_oom_waitq, &owait.wait);
1566         }
1567         mutex_lock(&memcg_oom_mutex);
1568         mem_cgroup_oom_unlock(mem);
1569         memcg_wakeup_oom(mem);
1570         mutex_unlock(&memcg_oom_mutex);
1571
1572         if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1573                 return false;
1574         /* Give chance to dying process */
1575         schedule_timeout(1);
1576         return true;
1577 }
1578
1579 /*
1580  * Currently used to update mapped file statistics, but the routine can be
1581  * generalized to update other statistics as well.
1582  *
1583  * Notes: Race condition
1584  *
1585  * We usually use page_cgroup_lock() for accessing page_cgroup member but
1586  * it tends to be costly. But considering some conditions, we doesn't need
1587  * to do so _always_.
1588  *
1589  * Considering "charge", lock_page_cgroup() is not required because all
1590  * file-stat operations happen after a page is attached to radix-tree. There
1591  * are no race with "charge".
1592  *
1593  * Considering "uncharge", we know that memcg doesn't clear pc->mem_cgroup
1594  * at "uncharge" intentionally. So, we always see valid pc->mem_cgroup even
1595  * if there are race with "uncharge". Statistics itself is properly handled
1596  * by flags.
1597  *
1598  * Considering "move", this is an only case we see a race. To make the race
1599  * small, we check MEM_CGROUP_ON_MOVE percpu value and detect there are
1600  * possibility of race condition. If there is, we take a lock.
1601  */
1602
1603 static void mem_cgroup_update_file_stat(struct page *page, int idx, int val)
1604 {
1605         struct mem_cgroup *mem;
1606         struct page_cgroup *pc = lookup_page_cgroup(page);
1607         bool need_unlock = false;
1608
1609         if (unlikely(!pc))
1610                 return;
1611
1612         rcu_read_lock();
1613         mem = pc->mem_cgroup;
1614         if (unlikely(!mem || !PageCgroupUsed(pc)))
1615                 goto out;
1616         /* pc->mem_cgroup is unstable ? */
1617         if (unlikely(mem_cgroup_stealed(mem))) {
1618                 /* take a lock against to access pc->mem_cgroup */
1619                 lock_page_cgroup(pc);
1620                 need_unlock = true;
1621                 mem = pc->mem_cgroup;
1622                 if (!mem || !PageCgroupUsed(pc))
1623                         goto out;
1624         }
1625
1626         this_cpu_add(mem->stat->count[idx], val);
1627
1628         switch (idx) {
1629         case MEM_CGROUP_STAT_FILE_MAPPED:
1630                 if (val > 0)
1631                         SetPageCgroupFileMapped(pc);
1632                 else if (!page_mapped(page))
1633                         ClearPageCgroupFileMapped(pc);
1634                 break;
1635         default:
1636                 BUG();
1637         }
1638
1639 out:
1640         if (unlikely(need_unlock))
1641                 unlock_page_cgroup(pc);
1642         rcu_read_unlock();
1643         return;
1644 }
1645
1646 void mem_cgroup_update_file_mapped(struct page *page, int val)
1647 {
1648         mem_cgroup_update_file_stat(page, MEM_CGROUP_STAT_FILE_MAPPED, val);
1649 }
1650
1651 /*
1652  * size of first charge trial. "32" comes from vmscan.c's magic value.
1653  * TODO: maybe necessary to use big numbers in big irons.
1654  */
1655 #define CHARGE_SIZE     (32 * PAGE_SIZE)
1656 struct memcg_stock_pcp {
1657         struct mem_cgroup *cached; /* this never be root cgroup */
1658         int charge;
1659         struct work_struct work;
1660 };
1661 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1662 static atomic_t memcg_drain_count;
1663
1664 /*
1665  * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1666  * from local stock and true is returned. If the stock is 0 or charges from a
1667  * cgroup which is not current target, returns false. This stock will be
1668  * refilled.
1669  */
1670 static bool consume_stock(struct mem_cgroup *mem)
1671 {
1672         struct memcg_stock_pcp *stock;
1673         bool ret = true;
1674
1675         stock = &get_cpu_var(memcg_stock);
1676         if (mem == stock->cached && stock->charge)
1677                 stock->charge -= PAGE_SIZE;
1678         else /* need to call res_counter_charge */
1679                 ret = false;
1680         put_cpu_var(memcg_stock);
1681         return ret;
1682 }
1683
1684 /*
1685  * Returns stocks cached in percpu to res_counter and reset cached information.
1686  */
1687 static void drain_stock(struct memcg_stock_pcp *stock)
1688 {
1689         struct mem_cgroup *old = stock->cached;
1690
1691         if (stock->charge) {
1692                 res_counter_uncharge(&old->res, stock->charge);
1693                 if (do_swap_account)
1694                         res_counter_uncharge(&old->memsw, stock->charge);
1695         }
1696         stock->cached = NULL;
1697         stock->charge = 0;
1698 }
1699
1700 /*
1701  * This must be called under preempt disabled or must be called by
1702  * a thread which is pinned to local cpu.
1703  */
1704 static void drain_local_stock(struct work_struct *dummy)
1705 {
1706         struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1707         drain_stock(stock);
1708 }
1709
1710 /*
1711  * Cache charges(val) which is from res_counter, to local per_cpu area.
1712  * This will be consumed by consume_stock() function, later.
1713  */
1714 static void refill_stock(struct mem_cgroup *mem, int val)
1715 {
1716         struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1717
1718         if (stock->cached != mem) { /* reset if necessary */
1719                 drain_stock(stock);
1720                 stock->cached = mem;
1721         }
1722         stock->charge += val;
1723         put_cpu_var(memcg_stock);
1724 }
1725
1726 /*
1727  * Tries to drain stocked charges in other cpus. This function is asynchronous
1728  * and just put a work per cpu for draining localy on each cpu. Caller can
1729  * expects some charges will be back to res_counter later but cannot wait for
1730  * it.
1731  */
1732 static void drain_all_stock_async(void)
1733 {
1734         int cpu;
1735         /* This function is for scheduling "drain" in asynchronous way.
1736          * The result of "drain" is not directly handled by callers. Then,
1737          * if someone is calling drain, we don't have to call drain more.
1738          * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1739          * there is a race. We just do loose check here.
1740          */
1741         if (atomic_read(&memcg_drain_count))
1742                 return;
1743         /* Notify other cpus that system-wide "drain" is running */
1744         atomic_inc(&memcg_drain_count);
1745         get_online_cpus();
1746         for_each_online_cpu(cpu) {
1747                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1748                 schedule_work_on(cpu, &stock->work);
1749         }
1750         put_online_cpus();
1751         atomic_dec(&memcg_drain_count);
1752         /* We don't wait for flush_work */
1753 }
1754
1755 /* This is a synchronous drain interface. */
1756 static void drain_all_stock_sync(void)
1757 {
1758         /* called when force_empty is called */
1759         atomic_inc(&memcg_drain_count);
1760         schedule_on_each_cpu(drain_local_stock);
1761         atomic_dec(&memcg_drain_count);
1762 }
1763
1764 /*
1765  * This function drains percpu counter value from DEAD cpu and
1766  * move it to local cpu. Note that this function can be preempted.
1767  */
1768 static void mem_cgroup_drain_pcp_counter(struct mem_cgroup *mem, int cpu)
1769 {
1770         int i;
1771
1772         spin_lock(&mem->pcp_counter_lock);
1773         for (i = 0; i < MEM_CGROUP_STAT_DATA; i++) {
1774                 s64 x = per_cpu(mem->stat->count[i], cpu);
1775
1776                 per_cpu(mem->stat->count[i], cpu) = 0;
1777                 mem->nocpu_base.count[i] += x;
1778         }
1779         /* need to clear ON_MOVE value, works as a kind of lock. */
1780         per_cpu(mem->stat->count[MEM_CGROUP_ON_MOVE], cpu) = 0;
1781         spin_unlock(&mem->pcp_counter_lock);
1782 }
1783
1784 static void synchronize_mem_cgroup_on_move(struct mem_cgroup *mem, int cpu)
1785 {
1786         int idx = MEM_CGROUP_ON_MOVE;
1787
1788         spin_lock(&mem->pcp_counter_lock);
1789         per_cpu(mem->stat->count[idx], cpu) = mem->nocpu_base.count[idx];
1790         spin_unlock(&mem->pcp_counter_lock);
1791 }
1792
1793 static int __cpuinit memcg_cpu_hotplug_callback(struct notifier_block *nb,
1794                                         unsigned long action,
1795                                         void *hcpu)
1796 {
1797         int cpu = (unsigned long)hcpu;
1798         struct memcg_stock_pcp *stock;
1799         struct mem_cgroup *iter;
1800
1801         if ((action == CPU_ONLINE)) {
1802                 for_each_mem_cgroup_all(iter)
1803                         synchronize_mem_cgroup_on_move(iter, cpu);
1804                 return NOTIFY_OK;
1805         }
1806
1807         if ((action != CPU_DEAD) || action != CPU_DEAD_FROZEN)
1808                 return NOTIFY_OK;
1809
1810         for_each_mem_cgroup_all(iter)
1811                 mem_cgroup_drain_pcp_counter(iter, cpu);
1812
1813         stock = &per_cpu(memcg_stock, cpu);
1814         drain_stock(stock);
1815         return NOTIFY_OK;
1816 }
1817
1818
1819 /* See __mem_cgroup_try_charge() for details */
1820 enum {
1821         CHARGE_OK,              /* success */
1822         CHARGE_RETRY,           /* need to retry but retry is not bad */
1823         CHARGE_NOMEM,           /* we can't do more. return -ENOMEM */
1824         CHARGE_WOULDBLOCK,      /* GFP_WAIT wasn't set and no enough res. */
1825         CHARGE_OOM_DIE,         /* the current is killed because of OOM */
1826 };
1827
1828 static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1829                                 int csize, bool oom_check)
1830 {
1831         struct mem_cgroup *mem_over_limit;
1832         struct res_counter *fail_res;
1833         unsigned long flags = 0;
1834         int ret;
1835
1836         ret = res_counter_charge(&mem->res, csize, &fail_res);
1837
1838         if (likely(!ret)) {
1839                 if (!do_swap_account)
1840                         return CHARGE_OK;
1841                 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1842                 if (likely(!ret))
1843                         return CHARGE_OK;
1844
1845                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1846                 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1847         } else
1848                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1849
1850         if (csize > PAGE_SIZE) /* change csize and retry */
1851                 return CHARGE_RETRY;
1852
1853         if (!(gfp_mask & __GFP_WAIT))
1854                 return CHARGE_WOULDBLOCK;
1855
1856         ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1857                                         gfp_mask, flags);
1858         /*
1859          * try_to_free_mem_cgroup_pages() might not give us a full
1860          * picture of reclaim. Some pages are reclaimed and might be
1861          * moved to swap cache or just unmapped from the cgroup.
1862          * Check the limit again to see if the reclaim reduced the
1863          * current usage of the cgroup before giving up
1864          */
1865         if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1866                 return CHARGE_RETRY;
1867
1868         /*
1869          * At task move, charge accounts can be doubly counted. So, it's
1870          * better to wait until the end of task_move if something is going on.
1871          */
1872         if (mem_cgroup_wait_acct_move(mem_over_limit))
1873                 return CHARGE_RETRY;
1874
1875         /* If we don't need to call oom-killer at el, return immediately */
1876         if (!oom_check)
1877                 return CHARGE_NOMEM;
1878         /* check OOM */
1879         if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1880                 return CHARGE_OOM_DIE;
1881
1882         return CHARGE_RETRY;
1883 }
1884
1885 /*
1886  * Unlike exported interface, "oom" parameter is added. if oom==true,
1887  * oom-killer can be invoked.
1888  */
1889 static int __mem_cgroup_try_charge(struct mm_struct *mm,
1890                 gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom)
1891 {
1892         int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1893         struct mem_cgroup *mem = NULL;
1894         int ret;
1895         int csize = CHARGE_SIZE;
1896
1897         /*
1898          * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1899          * in system level. So, allow to go ahead dying process in addition to
1900          * MEMDIE process.
1901          */
1902         if (unlikely(test_thread_flag(TIF_MEMDIE)
1903                      || fatal_signal_pending(current)))
1904                 goto bypass;
1905
1906         /*
1907          * We always charge the cgroup the mm_struct belongs to.
1908          * The mm_struct's mem_cgroup changes on task migration if the
1909          * thread group leader migrates. It's possible that mm is not
1910          * set, if so charge the init_mm (happens for pagecache usage).
1911          */
1912         if (!*memcg && !mm)
1913                 goto bypass;
1914 again:
1915         if (*memcg) { /* css should be a valid one */
1916                 mem = *memcg;
1917                 VM_BUG_ON(css_is_removed(&mem->css));
1918                 if (mem_cgroup_is_root(mem))
1919                         goto done;
1920                 if (consume_stock(mem))
1921                         goto done;
1922                 css_get(&mem->css);
1923         } else {
1924                 struct task_struct *p;
1925
1926                 rcu_read_lock();
1927                 p = rcu_dereference(mm->owner);
1928                 VM_BUG_ON(!p);
1929                 /*
1930                  * because we don't have task_lock(), "p" can exit while
1931                  * we're here. In that case, "mem" can point to root
1932                  * cgroup but never be NULL. (and task_struct itself is freed
1933                  * by RCU, cgroup itself is RCU safe.) Then, we have small
1934                  * risk here to get wrong cgroup. But such kind of mis-account
1935                  * by race always happens because we don't have cgroup_mutex().
1936                  * It's overkill and we allow that small race, here.
1937                  */
1938                 mem = mem_cgroup_from_task(p);
1939                 VM_BUG_ON(!mem);
1940                 if (mem_cgroup_is_root(mem)) {
1941                         rcu_read_unlock();
1942                         goto done;
1943                 }
1944                 if (consume_stock(mem)) {
1945                         /*
1946                          * It seems dagerous to access memcg without css_get().
1947                          * But considering how consume_stok works, it's not
1948                          * necessary. If consume_stock success, some charges
1949                          * from this memcg are cached on this cpu. So, we
1950                          * don't need to call css_get()/css_tryget() before
1951                          * calling consume_stock().
1952                          */
1953                         rcu_read_unlock();
1954                         goto done;
1955                 }
1956                 /* after here, we may be blocked. we need to get refcnt */
1957                 if (!css_tryget(&mem->css)) {
1958                         rcu_read_unlock();
1959                         goto again;
1960                 }
1961                 rcu_read_unlock();
1962         }
1963
1964         do {
1965                 bool oom_check;
1966
1967                 /* If killed, bypass charge */
1968                 if (fatal_signal_pending(current)) {
1969                         css_put(&mem->css);
1970                         goto bypass;
1971                 }
1972
1973                 oom_check = false;
1974                 if (oom && !nr_oom_retries) {
1975                         oom_check = true;
1976                         nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1977                 }
1978
1979                 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1980
1981                 switch (ret) {
1982                 case CHARGE_OK:
1983                         break;
1984                 case CHARGE_RETRY: /* not in OOM situation but retry */
1985                         csize = PAGE_SIZE;
1986                         css_put(&mem->css);
1987                         mem = NULL;
1988                         goto again;
1989                 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
1990                         css_put(&mem->css);
1991                         goto nomem;
1992                 case CHARGE_NOMEM: /* OOM routine works */
1993                         if (!oom) {
1994                                 css_put(&mem->css);
1995                                 goto nomem;
1996                         }
1997                         /* If oom, we never return -ENOMEM */
1998                         nr_oom_retries--;
1999                         break;
2000                 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
2001                         css_put(&mem->css);
2002                         goto bypass;
2003                 }
2004         } while (ret != CHARGE_OK);
2005
2006         if (csize > PAGE_SIZE)
2007                 refill_stock(mem, csize - PAGE_SIZE);
2008         css_put(&mem->css);
2009 done:
2010         *memcg = mem;
2011         return 0;
2012 nomem:
2013         *memcg = NULL;
2014         return -ENOMEM;
2015 bypass:
2016         *memcg = NULL;
2017         return 0;
2018 }
2019
2020 /*
2021  * Somemtimes we have to undo a charge we got by try_charge().
2022  * This function is for that and do uncharge, put css's refcnt.
2023  * gotten by try_charge().
2024  */
2025 static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
2026                                                         unsigned long count)
2027 {
2028         if (!mem_cgroup_is_root(mem)) {
2029                 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
2030                 if (do_swap_account)
2031                         res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
2032         }
2033 }
2034
2035 static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
2036 {
2037         __mem_cgroup_cancel_charge(mem, 1);
2038 }
2039
2040 /*
2041  * A helper function to get mem_cgroup from ID. must be called under
2042  * rcu_read_lock(). The caller must check css_is_removed() or some if
2043  * it's concern. (dropping refcnt from swap can be called against removed
2044  * memcg.)
2045  */
2046 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
2047 {
2048         struct cgroup_subsys_state *css;
2049
2050         /* ID 0 is unused ID */
2051         if (!id)
2052                 return NULL;
2053         css = css_lookup(&mem_cgroup_subsys, id);
2054         if (!css)
2055                 return NULL;
2056         return container_of(css, struct mem_cgroup, css);
2057 }
2058
2059 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
2060 {
2061         struct mem_cgroup *mem = NULL;
2062         struct page_cgroup *pc;
2063         unsigned short id;
2064         swp_entry_t ent;
2065
2066         VM_BUG_ON(!PageLocked(page));
2067
2068         pc = lookup_page_cgroup(page);
2069         lock_page_cgroup(pc);
2070         if (PageCgroupUsed(pc)) {
2071                 mem = pc->mem_cgroup;
2072                 if (mem && !css_tryget(&mem->css))
2073                         mem = NULL;
2074         } else if (PageSwapCache(page)) {
2075                 ent.val = page_private(page);
2076                 id = lookup_swap_cgroup(ent);
2077                 rcu_read_lock();
2078                 mem = mem_cgroup_lookup(id);
2079                 if (mem && !css_tryget(&mem->css))
2080                         mem = NULL;
2081                 rcu_read_unlock();
2082         }
2083         unlock_page_cgroup(pc);
2084         return mem;
2085 }
2086
2087 /*
2088  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
2089  * USED state. If already USED, uncharge and return.
2090  */
2091
2092 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
2093                                      struct page_cgroup *pc,
2094                                      enum charge_type ctype)
2095 {
2096         /* try_charge() can return NULL to *memcg, taking care of it. */
2097         if (!mem)
2098                 return;
2099
2100         lock_page_cgroup(pc);
2101         if (unlikely(PageCgroupUsed(pc))) {
2102                 unlock_page_cgroup(pc);
2103                 mem_cgroup_cancel_charge(mem);
2104                 return;
2105         }
2106
2107         pc->mem_cgroup = mem;
2108         /*
2109          * We access a page_cgroup asynchronously without lock_page_cgroup().
2110          * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
2111          * is accessed after testing USED bit. To make pc->mem_cgroup visible
2112          * before USED bit, we need memory barrier here.
2113          * See mem_cgroup_add_lru_list(), etc.
2114          */
2115         smp_wmb();
2116         switch (ctype) {
2117         case MEM_CGROUP_CHARGE_TYPE_CACHE:
2118         case MEM_CGROUP_CHARGE_TYPE_SHMEM:
2119                 SetPageCgroupCache(pc);
2120                 SetPageCgroupUsed(pc);
2121                 break;
2122         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2123                 ClearPageCgroupCache(pc);
2124                 SetPageCgroupUsed(pc);
2125                 break;
2126         default:
2127                 break;
2128         }
2129
2130         mem_cgroup_charge_statistics(mem, pc, true);
2131
2132         unlock_page_cgroup(pc);
2133         /*
2134          * "charge_statistics" updated event counter. Then, check it.
2135          * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
2136          * if they exceeds softlimit.
2137          */
2138         memcg_check_events(mem, pc->page);
2139 }
2140
2141 /**
2142  * __mem_cgroup_move_account - move account of the page
2143  * @pc: page_cgroup of the page.
2144  * @from: mem_cgroup which the page is moved from.
2145  * @to: mem_cgroup which the page is moved to. @from != @to.
2146  * @uncharge: whether we should call uncharge and css_put against @from.
2147  *
2148  * The caller must confirm following.
2149  * - page is not on LRU (isolate_page() is useful.)
2150  * - the pc is locked, used, and ->mem_cgroup points to @from.
2151  *
2152  * This function doesn't do "charge" nor css_get to new cgroup. It should be
2153  * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
2154  * true, this function does "uncharge" from old cgroup, but it doesn't if
2155  * @uncharge is false, so a caller should do "uncharge".
2156  */
2157
2158 static void __mem_cgroup_move_account(struct page_cgroup *pc,
2159         struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
2160 {
2161         VM_BUG_ON(from == to);
2162         VM_BUG_ON(PageLRU(pc->page));
2163         VM_BUG_ON(!page_is_cgroup_locked(pc));
2164         VM_BUG_ON(!PageCgroupUsed(pc));
2165         VM_BUG_ON(pc->mem_cgroup != from);
2166
2167         if (PageCgroupFileMapped(pc)) {
2168                 /* Update mapped_file data for mem_cgroup */
2169                 preempt_disable();
2170                 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2171                 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
2172                 preempt_enable();
2173         }
2174         mem_cgroup_charge_statistics(from, pc, false);
2175         if (uncharge)
2176                 /* This is not "cancel", but cancel_charge does all we need. */
2177                 mem_cgroup_cancel_charge(from);
2178
2179         /* caller should have done css_get */
2180         pc->mem_cgroup = to;
2181         mem_cgroup_charge_statistics(to, pc, true);
2182         /*
2183          * We charges against "to" which may not have any tasks. Then, "to"
2184          * can be under rmdir(). But in current implementation, caller of
2185          * this function is just force_empty() and move charge, so it's
2186          * garanteed that "to" is never removed. So, we don't check rmdir
2187          * status here.
2188          */
2189 }
2190
2191 /*
2192  * check whether the @pc is valid for moving account and call
2193  * __mem_cgroup_move_account()
2194  */
2195 static int mem_cgroup_move_account(struct page_cgroup *pc,
2196                 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
2197 {
2198         int ret = -EINVAL;
2199         lock_page_cgroup(pc);
2200         if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
2201                 __mem_cgroup_move_account(pc, from, to, uncharge);
2202                 ret = 0;
2203         }
2204         unlock_page_cgroup(pc);
2205         /*
2206          * check events
2207          */
2208         memcg_check_events(to, pc->page);
2209         memcg_check_events(from, pc->page);
2210         return ret;
2211 }
2212
2213 /*
2214  * move charges to its parent.
2215  */
2216
2217 static int mem_cgroup_move_parent(struct page_cgroup *pc,
2218                                   struct mem_cgroup *child,
2219                                   gfp_t gfp_mask)
2220 {
2221         struct page *page = pc->page;
2222         struct cgroup *cg = child->css.cgroup;
2223         struct cgroup *pcg = cg->parent;
2224         struct mem_cgroup *parent;
2225         int ret;
2226
2227         /* Is ROOT ? */
2228         if (!pcg)
2229                 return -EINVAL;
2230
2231         ret = -EBUSY;
2232         if (!get_page_unless_zero(page))
2233                 goto out;
2234         if (isolate_lru_page(page))
2235                 goto put;
2236
2237         parent = mem_cgroup_from_cont(pcg);
2238         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
2239         if (ret || !parent)
2240                 goto put_back;
2241
2242         ret = mem_cgroup_move_account(pc, child, parent, true);
2243         if (ret)
2244                 mem_cgroup_cancel_charge(parent);
2245 put_back:
2246         putback_lru_page(page);
2247 put:
2248         put_page(page);
2249 out:
2250         return ret;
2251 }
2252
2253 /*
2254  * Charge the memory controller for page usage.
2255  * Return
2256  * 0 if the charge was successful
2257  * < 0 if the cgroup is over its limit
2258  */
2259 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2260                                 gfp_t gfp_mask, enum charge_type ctype)
2261 {
2262         struct mem_cgroup *mem = NULL;
2263         struct page_cgroup *pc;
2264         int ret;
2265
2266         pc = lookup_page_cgroup(page);
2267         /* can happen at boot */
2268         if (unlikely(!pc))
2269                 return 0;
2270         prefetchw(pc);
2271
2272         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
2273         if (ret || !mem)
2274                 return ret;
2275
2276         __mem_cgroup_commit_charge(mem, pc, ctype);
2277         return 0;
2278 }
2279
2280 int mem_cgroup_newpage_charge(struct page *page,
2281                               struct mm_struct *mm, gfp_t gfp_mask)
2282 {
2283         if (mem_cgroup_disabled())
2284                 return 0;
2285         if (PageCompound(page))
2286                 return 0;
2287         /*
2288          * If already mapped, we don't have to account.
2289          * If page cache, page->mapping has address_space.
2290          * But page->mapping may have out-of-use anon_vma pointer,
2291          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2292          * is NULL.
2293          */
2294         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2295                 return 0;
2296         if (unlikely(!mm))
2297                 mm = &init_mm;
2298         return mem_cgroup_charge_common(page, mm, gfp_mask,
2299                                 MEM_CGROUP_CHARGE_TYPE_MAPPED);
2300 }
2301
2302 static void
2303 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2304                                         enum charge_type ctype);
2305
2306 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2307                                 gfp_t gfp_mask)
2308 {
2309         int ret;
2310
2311         if (mem_cgroup_disabled())
2312                 return 0;
2313         if (PageCompound(page))
2314                 return 0;
2315         /*
2316          * Corner case handling. This is called from add_to_page_cache()
2317          * in usual. But some FS (shmem) precharges this page before calling it
2318          * and call add_to_page_cache() with GFP_NOWAIT.
2319          *
2320          * For GFP_NOWAIT case, the page may be pre-charged before calling
2321          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2322          * charge twice. (It works but has to pay a bit larger cost.)
2323          * And when the page is SwapCache, it should take swap information
2324          * into account. This is under lock_page() now.
2325          */
2326         if (!(gfp_mask & __GFP_WAIT)) {
2327                 struct page_cgroup *pc;
2328
2329                 pc = lookup_page_cgroup(page);
2330                 if (!pc)
2331                         return 0;
2332                 lock_page_cgroup(pc);
2333                 if (PageCgroupUsed(pc)) {
2334                         unlock_page_cgroup(pc);
2335                         return 0;
2336                 }
2337                 unlock_page_cgroup(pc);
2338         }
2339
2340         if (unlikely(!mm))
2341                 mm = &init_mm;
2342
2343         if (page_is_file_cache(page))
2344                 return mem_cgroup_charge_common(page, mm, gfp_mask,
2345                                 MEM_CGROUP_CHARGE_TYPE_CACHE);
2346
2347         /* shmem */
2348         if (PageSwapCache(page)) {
2349                 struct mem_cgroup *mem = NULL;
2350
2351                 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2352                 if (!ret)
2353                         __mem_cgroup_commit_charge_swapin(page, mem,
2354                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
2355         } else
2356                 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
2357                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
2358
2359         return ret;
2360 }
2361
2362 /*
2363  * While swap-in, try_charge -> commit or cancel, the page is locked.
2364  * And when try_charge() successfully returns, one refcnt to memcg without
2365  * struct page_cgroup is acquired. This refcnt will be consumed by
2366  * "commit()" or removed by "cancel()"
2367  */
2368 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2369                                  struct page *page,
2370                                  gfp_t mask, struct mem_cgroup **ptr)
2371 {
2372         struct mem_cgroup *mem;
2373         int ret;
2374
2375         if (mem_cgroup_disabled())
2376                 return 0;
2377
2378         if (!do_swap_account)
2379                 goto charge_cur_mm;
2380         /*
2381          * A racing thread's fault, or swapoff, may have already updated
2382          * the pte, and even removed page from swap cache: in those cases
2383          * do_swap_page()'s pte_same() test will fail; but there's also a
2384          * KSM case which does need to charge the page.
2385          */
2386         if (!PageSwapCache(page))
2387                 goto charge_cur_mm;
2388         mem = try_get_mem_cgroup_from_page(page);
2389         if (!mem)
2390                 goto charge_cur_mm;
2391         *ptr = mem;
2392         ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
2393         css_put(&mem->css);
2394         return ret;
2395 charge_cur_mm:
2396         if (unlikely(!mm))
2397                 mm = &init_mm;
2398         return __mem_cgroup_try_charge(mm, mask, ptr, true);
2399 }
2400
2401 static void
2402 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2403                                         enum charge_type ctype)
2404 {
2405         struct page_cgroup *pc;
2406
2407         if (mem_cgroup_disabled())
2408                 return;
2409         if (!ptr)
2410                 return;
2411         cgroup_exclude_rmdir(&ptr->css);
2412         pc = lookup_page_cgroup(page);
2413         mem_cgroup_lru_del_before_commit_swapcache(page);
2414         __mem_cgroup_commit_charge(ptr, pc, ctype);
2415         mem_cgroup_lru_add_after_commit_swapcache(page);
2416         /*
2417          * Now swap is on-memory. This means this page may be
2418          * counted both as mem and swap....double count.
2419          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2420          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2421          * may call delete_from_swap_cache() before reach here.
2422          */
2423         if (do_swap_account && PageSwapCache(page)) {
2424                 swp_entry_t ent = {.val = page_private(page)};
2425                 unsigned short id;
2426                 struct mem_cgroup *memcg;
2427
2428                 id = swap_cgroup_record(ent, 0);
2429                 rcu_read_lock();
2430                 memcg = mem_cgroup_lookup(id);
2431                 if (memcg) {
2432                         /*
2433                          * This recorded memcg can be obsolete one. So, avoid
2434                          * calling css_tryget
2435                          */
2436                         if (!mem_cgroup_is_root(memcg))
2437                                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2438                         mem_cgroup_swap_statistics(memcg, false);
2439                         mem_cgroup_put(memcg);
2440                 }
2441                 rcu_read_unlock();
2442         }
2443         /*
2444          * At swapin, we may charge account against cgroup which has no tasks.
2445          * So, rmdir()->pre_destroy() can be called while we do this charge.
2446          * In that case, we need to call pre_destroy() again. check it here.
2447          */
2448         cgroup_release_and_wakeup_rmdir(&ptr->css);
2449 }
2450
2451 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2452 {
2453         __mem_cgroup_commit_charge_swapin(page, ptr,
2454                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
2455 }
2456
2457 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2458 {
2459         if (mem_cgroup_disabled())
2460                 return;
2461         if (!mem)
2462                 return;
2463         mem_cgroup_cancel_charge(mem);
2464 }
2465
2466 static void
2467 __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
2468 {
2469         struct memcg_batch_info *batch = NULL;
2470         bool uncharge_memsw = true;
2471         /* If swapout, usage of swap doesn't decrease */
2472         if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2473                 uncharge_memsw = false;
2474
2475         batch = &current->memcg_batch;
2476         /*
2477          * In usual, we do css_get() when we remember memcg pointer.
2478          * But in this case, we keep res->usage until end of a series of
2479          * uncharges. Then, it's ok to ignore memcg's refcnt.
2480          */
2481         if (!batch->memcg)
2482                 batch->memcg = mem;
2483         /*
2484          * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2485          * In those cases, all pages freed continously can be expected to be in
2486          * the same cgroup and we have chance to coalesce uncharges.
2487          * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2488          * because we want to do uncharge as soon as possible.
2489          */
2490
2491         if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2492                 goto direct_uncharge;
2493
2494         /*
2495          * In typical case, batch->memcg == mem. This means we can
2496          * merge a series of uncharges to an uncharge of res_counter.
2497          * If not, we uncharge res_counter ony by one.
2498          */
2499         if (batch->memcg != mem)
2500                 goto direct_uncharge;
2501         /* remember freed charge and uncharge it later */
2502         batch->bytes += PAGE_SIZE;
2503         if (uncharge_memsw)
2504                 batch->memsw_bytes += PAGE_SIZE;
2505         return;
2506 direct_uncharge:
2507         res_counter_uncharge(&mem->res, PAGE_SIZE);
2508         if (uncharge_memsw)
2509                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
2510         if (unlikely(batch->memcg != mem))
2511                 memcg_oom_recover(mem);
2512         return;
2513 }
2514
2515 /*
2516  * uncharge if !page_mapped(page)
2517  */
2518 static struct mem_cgroup *
2519 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2520 {
2521         struct page_cgroup *pc;
2522         struct mem_cgroup *mem = NULL;
2523
2524         if (mem_cgroup_disabled())
2525                 return NULL;
2526
2527         if (PageSwapCache(page))
2528                 return NULL;
2529
2530         /*
2531          * Check if our page_cgroup is valid
2532          */
2533         pc = lookup_page_cgroup(page);
2534         if (unlikely(!pc || !PageCgroupUsed(pc)))
2535                 return NULL;
2536
2537         lock_page_cgroup(pc);
2538
2539         mem = pc->mem_cgroup;
2540
2541         if (!PageCgroupUsed(pc))
2542                 goto unlock_out;
2543
2544         switch (ctype) {
2545         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2546         case MEM_CGROUP_CHARGE_TYPE_DROP:
2547                 /* See mem_cgroup_prepare_migration() */
2548                 if (page_mapped(page) || PageCgroupMigration(pc))
2549                         goto unlock_out;
2550                 break;
2551         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2552                 if (!PageAnon(page)) {  /* Shared memory */
2553                         if (page->mapping && !page_is_file_cache(page))
2554                                 goto unlock_out;
2555                 } else if (page_mapped(page)) /* Anon */
2556                                 goto unlock_out;
2557                 break;
2558         default:
2559                 break;
2560         }
2561
2562         mem_cgroup_charge_statistics(mem, pc, false);
2563
2564         ClearPageCgroupUsed(pc);
2565         /*
2566          * pc->mem_cgroup is not cleared here. It will be accessed when it's
2567          * freed from LRU. This is safe because uncharged page is expected not
2568          * to be reused (freed soon). Exception is SwapCache, it's handled by
2569          * special functions.
2570          */
2571
2572         unlock_page_cgroup(pc);
2573         /*
2574          * even after unlock, we have mem->res.usage here and this memcg
2575          * will never be freed.
2576          */
2577         memcg_check_events(mem, page);
2578         if (do_swap_account && ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT) {
2579                 mem_cgroup_swap_statistics(mem, true);
2580                 mem_cgroup_get(mem);
2581         }
2582         if (!mem_cgroup_is_root(mem))
2583                 __do_uncharge(mem, ctype);
2584
2585         return mem;
2586
2587 unlock_out:
2588         unlock_page_cgroup(pc);
2589         return NULL;
2590 }
2591
2592 void mem_cgroup_uncharge_page(struct page *page)
2593 {
2594         /* early check. */
2595         if (page_mapped(page))
2596                 return;
2597         if (page->mapping && !PageAnon(page))
2598                 return;
2599         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2600 }
2601
2602 void mem_cgroup_uncharge_cache_page(struct page *page)
2603 {
2604         VM_BUG_ON(page_mapped(page));
2605         VM_BUG_ON(page->mapping);
2606         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2607 }
2608
2609 /*
2610  * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2611  * In that cases, pages are freed continuously and we can expect pages
2612  * are in the same memcg. All these calls itself limits the number of
2613  * pages freed at once, then uncharge_start/end() is called properly.
2614  * This may be called prural(2) times in a context,
2615  */
2616
2617 void mem_cgroup_uncharge_start(void)
2618 {
2619         current->memcg_batch.do_batch++;
2620         /* We can do nest. */
2621         if (current->memcg_batch.do_batch == 1) {
2622                 current->memcg_batch.memcg = NULL;
2623                 current->memcg_batch.bytes = 0;
2624                 current->memcg_batch.memsw_bytes = 0;
2625         }
2626 }
2627
2628 void mem_cgroup_uncharge_end(void)
2629 {
2630         struct memcg_batch_info *batch = &current->memcg_batch;
2631
2632         if (!batch->do_batch)
2633                 return;
2634
2635         batch->do_batch--;
2636         if (batch->do_batch) /* If stacked, do nothing. */
2637                 return;
2638
2639         if (!batch->memcg)
2640                 return;
2641         /*
2642          * This "batch->memcg" is valid without any css_get/put etc...
2643          * bacause we hide charges behind us.
2644          */
2645         if (batch->bytes)
2646                 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2647         if (batch->memsw_bytes)
2648                 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
2649         memcg_oom_recover(batch->memcg);
2650         /* forget this pointer (for sanity check) */
2651         batch->memcg = NULL;
2652 }
2653
2654 #ifdef CONFIG_SWAP
2655 /*
2656  * called after __delete_from_swap_cache() and drop "page" account.
2657  * memcg information is recorded to swap_cgroup of "ent"
2658  */
2659 void
2660 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
2661 {
2662         struct mem_cgroup *memcg;
2663         int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
2664
2665         if (!swapout) /* this was a swap cache but the swap is unused ! */
2666                 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2667
2668         memcg = __mem_cgroup_uncharge_common(page, ctype);
2669
2670         /*
2671          * record memcg information,  if swapout && memcg != NULL,
2672          * mem_cgroup_get() was called in uncharge().
2673          */
2674         if (do_swap_account && swapout && memcg)
2675                 swap_cgroup_record(ent, css_id(&memcg->css));
2676 }
2677 #endif
2678
2679 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2680 /*
2681  * called from swap_entry_free(). remove record in swap_cgroup and
2682  * uncharge "memsw" account.
2683  */
2684 void mem_cgroup_uncharge_swap(swp_entry_t ent)
2685 {
2686         struct mem_cgroup *memcg;
2687         unsigned short id;
2688
2689         if (!do_swap_account)
2690                 return;
2691
2692         id = swap_cgroup_record(ent, 0);
2693         rcu_read_lock();
2694         memcg = mem_cgroup_lookup(id);
2695         if (memcg) {
2696                 /*
2697                  * We uncharge this because swap is freed.
2698                  * This memcg can be obsolete one. We avoid calling css_tryget
2699                  */
2700                 if (!mem_cgroup_is_root(memcg))
2701                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2702                 mem_cgroup_swap_statistics(memcg, false);
2703                 mem_cgroup_put(memcg);
2704         }
2705         rcu_read_unlock();
2706 }
2707
2708 /**
2709  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2710  * @entry: swap entry to be moved
2711  * @from:  mem_cgroup which the entry is moved from
2712  * @to:  mem_cgroup which the entry is moved to
2713  * @need_fixup: whether we should fixup res_counters and refcounts.
2714  *
2715  * It succeeds only when the swap_cgroup's record for this entry is the same
2716  * as the mem_cgroup's id of @from.
2717  *
2718  * Returns 0 on success, -EINVAL on failure.
2719  *
2720  * The caller must have charged to @to, IOW, called res_counter_charge() about
2721  * both res and memsw, and called css_get().
2722  */
2723 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2724                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2725 {
2726         unsigned short old_id, new_id;
2727
2728         old_id = css_id(&from->css);
2729         new_id = css_id(&to->css);
2730
2731         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2732                 mem_cgroup_swap_statistics(from, false);
2733                 mem_cgroup_swap_statistics(to, true);
2734                 /*
2735                  * This function is only called from task migration context now.
2736                  * It postpones res_counter and refcount handling till the end
2737                  * of task migration(mem_cgroup_clear_mc()) for performance
2738                  * improvement. But we cannot postpone mem_cgroup_get(to)
2739                  * because if the process that has been moved to @to does
2740                  * swap-in, the refcount of @to might be decreased to 0.
2741                  */
2742                 mem_cgroup_get(to);
2743                 if (need_fixup) {
2744                         if (!mem_cgroup_is_root(from))
2745                                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2746                         mem_cgroup_put(from);
2747                         /*
2748                          * we charged both to->res and to->memsw, so we should
2749                          * uncharge to->res.
2750                          */
2751                         if (!mem_cgroup_is_root(to))
2752                                 res_counter_uncharge(&to->res, PAGE_SIZE);
2753                 }
2754                 return 0;
2755         }
2756         return -EINVAL;
2757 }
2758 #else
2759 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2760                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2761 {
2762         return -EINVAL;
2763 }
2764 #endif
2765
2766 /*
2767  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2768  * page belongs to.
2769  */
2770 int mem_cgroup_prepare_migration(struct page *page,
2771         struct page *newpage, struct mem_cgroup **ptr)
2772 {
2773         struct page_cgroup *pc;
2774         struct mem_cgroup *mem = NULL;
2775         enum charge_type ctype;
2776         int ret = 0;
2777
2778         if (mem_cgroup_disabled())
2779                 return 0;
2780
2781         pc = lookup_page_cgroup(page);
2782         lock_page_cgroup(pc);
2783         if (PageCgroupUsed(pc)) {
2784                 mem = pc->mem_cgroup;
2785                 css_get(&mem->css);
2786                 /*
2787                  * At migrating an anonymous page, its mapcount goes down
2788                  * to 0 and uncharge() will be called. But, even if it's fully
2789                  * unmapped, migration may fail and this page has to be
2790                  * charged again. We set MIGRATION flag here and delay uncharge
2791                  * until end_migration() is called
2792                  *
2793                  * Corner Case Thinking
2794                  * A)
2795                  * When the old page was mapped as Anon and it's unmap-and-freed
2796                  * while migration was ongoing.
2797                  * If unmap finds the old page, uncharge() of it will be delayed
2798                  * until end_migration(). If unmap finds a new page, it's
2799                  * uncharged when it make mapcount to be 1->0. If unmap code
2800                  * finds swap_migration_entry, the new page will not be mapped
2801                  * and end_migration() will find it(mapcount==0).
2802                  *
2803                  * B)
2804                  * When the old page was mapped but migraion fails, the kernel
2805                  * remaps it. A charge for it is kept by MIGRATION flag even
2806                  * if mapcount goes down to 0. We can do remap successfully
2807                  * without charging it again.
2808                  *
2809                  * C)
2810                  * The "old" page is under lock_page() until the end of
2811                  * migration, so, the old page itself will not be swapped-out.
2812                  * If the new page is swapped out before end_migraton, our
2813                  * hook to usual swap-out path will catch the event.
2814                  */
2815                 if (PageAnon(page))
2816                         SetPageCgroupMigration(pc);
2817         }
2818         unlock_page_cgroup(pc);
2819         /*
2820          * If the page is not charged at this point,
2821          * we return here.
2822          */
2823         if (!mem)
2824                 return 0;
2825
2826         *ptr = mem;
2827         ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
2828         css_put(&mem->css);/* drop extra refcnt */
2829         if (ret || *ptr == NULL) {
2830                 if (PageAnon(page)) {
2831                         lock_page_cgroup(pc);
2832                         ClearPageCgroupMigration(pc);
2833                         unlock_page_cgroup(pc);
2834                         /*
2835                          * The old page may be fully unmapped while we kept it.
2836                          */
2837                         mem_cgroup_uncharge_page(page);
2838                 }
2839                 return -ENOMEM;
2840         }
2841         /*
2842          * We charge new page before it's used/mapped. So, even if unlock_page()
2843          * is called before end_migration, we can catch all events on this new
2844          * page. In the case new page is migrated but not remapped, new page's
2845          * mapcount will be finally 0 and we call uncharge in end_migration().
2846          */
2847         pc = lookup_page_cgroup(newpage);
2848         if (PageAnon(page))
2849                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2850         else if (page_is_file_cache(page))
2851                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2852         else
2853                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
2854         __mem_cgroup_commit_charge(mem, pc, ctype);
2855         return ret;
2856 }
2857
2858 /* remove redundant charge if migration failed*/
2859 void mem_cgroup_end_migration(struct mem_cgroup *mem,
2860         struct page *oldpage, struct page *newpage)
2861 {
2862         struct page *used, *unused;
2863         struct page_cgroup *pc;
2864
2865         if (!mem)
2866                 return;
2867         /* blocks rmdir() */
2868         cgroup_exclude_rmdir(&mem->css);
2869         /* at migration success, oldpage->mapping is NULL. */
2870         if (oldpage->mapping) {
2871                 used = oldpage;
2872                 unused = newpage;
2873         } else {
2874                 used = newpage;
2875                 unused = oldpage;
2876         }
2877         /*
2878          * We disallowed uncharge of pages under migration because mapcount
2879          * of the page goes down to zero, temporarly.
2880          * Clear the flag and check the page should be charged.
2881          */
2882         pc = lookup_page_cgroup(oldpage);
2883         lock_page_cgroup(pc);
2884         ClearPageCgroupMigration(pc);
2885         unlock_page_cgroup(pc);
2886
2887         __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2888
2889         /*
2890          * If a page is a file cache, radix-tree replacement is very atomic
2891          * and we can skip this check. When it was an Anon page, its mapcount
2892          * goes down to 0. But because we added MIGRATION flage, it's not
2893          * uncharged yet. There are several case but page->mapcount check
2894          * and USED bit check in mem_cgroup_uncharge_page() will do enough
2895          * check. (see prepare_charge() also)
2896          */
2897         if (PageAnon(used))
2898                 mem_cgroup_uncharge_page(used);
2899         /*
2900          * At migration, we may charge account against cgroup which has no
2901          * tasks.
2902          * So, rmdir()->pre_destroy() can be called while we do this charge.
2903          * In that case, we need to call pre_destroy() again. check it here.
2904          */
2905         cgroup_release_and_wakeup_rmdir(&mem->css);
2906 }
2907
2908 /*
2909  * A call to try to shrink memory usage on charge failure at shmem's swapin.
2910  * Calling hierarchical_reclaim is not enough because we should update
2911  * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2912  * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2913  * not from the memcg which this page would be charged to.
2914  * try_charge_swapin does all of these works properly.
2915  */
2916 int mem_cgroup_shmem_charge_fallback(struct page *page,
2917                             struct mm_struct *mm,
2918                             gfp_t gfp_mask)
2919 {
2920         struct mem_cgroup *mem = NULL;
2921         int ret;
2922
2923         if (mem_cgroup_disabled())
2924                 return 0;
2925
2926         ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2927         if (!ret)
2928                 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
2929
2930         return ret;
2931 }
2932
2933 static DEFINE_MUTEX(set_limit_mutex);
2934
2935 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2936                                 unsigned long long val)
2937 {
2938         int retry_count;
2939         u64 memswlimit, memlimit;
2940         int ret = 0;
2941         int children = mem_cgroup_count_children(memcg);
2942         u64 curusage, oldusage;
2943         int enlarge;
2944
2945         /*
2946          * For keeping hierarchical_reclaim simple, how long we should retry
2947          * is depends on callers. We set our retry-count to be function
2948          * of # of children which we should visit in this loop.
2949          */
2950         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
2951
2952         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2953
2954         enlarge = 0;
2955         while (retry_count) {
2956                 if (signal_pending(current)) {
2957                         ret = -EINTR;
2958                         break;
2959                 }
2960                 /*
2961                  * Rather than hide all in some function, I do this in
2962                  * open coded manner. You see what this really does.
2963                  * We have to guarantee mem->res.limit < mem->memsw.limit.
2964                  */
2965                 mutex_lock(&set_limit_mutex);
2966                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2967                 if (memswlimit < val) {
2968                         ret = -EINVAL;
2969                         mutex_unlock(&set_limit_mutex);
2970                         break;
2971                 }
2972
2973                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2974                 if (memlimit < val)
2975                         enlarge = 1;
2976
2977                 ret = res_counter_set_limit(&memcg->res, val);
2978                 if (!ret) {
2979                         if (memswlimit == val)
2980                                 memcg->memsw_is_minimum = true;
2981                         else
2982                                 memcg->memsw_is_minimum = false;
2983                 }
2984                 mutex_unlock(&set_limit_mutex);
2985
2986                 if (!ret)
2987                         break;
2988
2989                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
2990                                                 MEM_CGROUP_RECLAIM_SHRINK);
2991                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2992                 /* Usage is reduced ? */
2993                 if (curusage >= oldusage)
2994                         retry_count--;
2995                 else
2996                         oldusage = curusage;
2997         }
2998         if (!ret && enlarge)
2999                 memcg_oom_recover(memcg);
3000
3001         return ret;
3002 }
3003
3004 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
3005                                         unsigned long long val)
3006 {
3007         int retry_count;
3008         u64 memlimit, memswlimit, oldusage, curusage;
3009         int children = mem_cgroup_count_children(memcg);
3010         int ret = -EBUSY;
3011         int enlarge = 0;
3012
3013         /* see mem_cgroup_resize_res_limit */
3014         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
3015         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3016         while (retry_count) {
3017                 if (signal_pending(current)) {
3018                         ret = -EINTR;
3019                         break;
3020                 }
3021                 /*
3022                  * Rather than hide all in some function, I do this in
3023                  * open coded manner. You see what this really does.
3024                  * We have to guarantee mem->res.limit < mem->memsw.limit.
3025                  */
3026                 mutex_lock(&set_limit_mutex);
3027                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3028                 if (memlimit > val) {
3029                         ret = -EINVAL;
3030                         mutex_unlock(&set_limit_mutex);
3031                         break;
3032                 }
3033                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3034                 if (memswlimit < val)
3035                         enlarge = 1;
3036                 ret = res_counter_set_limit(&memcg->memsw, val);
3037                 if (!ret) {
3038                         if (memlimit == val)
3039                                 memcg->memsw_is_minimum = true;
3040                         else
3041                                 memcg->memsw_is_minimum = false;
3042                 }
3043                 mutex_unlock(&set_limit_mutex);
3044
3045                 if (!ret)
3046                         break;
3047
3048                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
3049                                                 MEM_CGROUP_RECLAIM_NOSWAP |
3050                                                 MEM_CGROUP_RECLAIM_SHRINK);
3051                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
3052                 /* Usage is reduced ? */
3053                 if (curusage >= oldusage)
3054                         retry_count--;
3055                 else
3056                         oldusage = curusage;
3057         }
3058         if (!ret && enlarge)
3059                 memcg_oom_recover(memcg);
3060         return ret;
3061 }
3062
3063 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
3064                                             gfp_t gfp_mask)
3065 {
3066         unsigned long nr_reclaimed = 0;
3067         struct mem_cgroup_per_zone *mz, *next_mz = NULL;
3068         unsigned long reclaimed;
3069         int loop = 0;
3070         struct mem_cgroup_tree_per_zone *mctz;
3071         unsigned long long excess;
3072
3073         if (order > 0)
3074                 return 0;
3075
3076         mctz = soft_limit_tree_node_zone(zone_to_nid(zone), zone_idx(zone));
3077         /*
3078          * This loop can run a while, specially if mem_cgroup's continuously
3079          * keep exceeding their soft limit and putting the system under
3080          * pressure
3081          */
3082         do {
3083                 if (next_mz)
3084                         mz = next_mz;
3085                 else
3086                         mz = mem_cgroup_largest_soft_limit_node(mctz);
3087                 if (!mz)
3088                         break;
3089
3090                 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
3091                                                 gfp_mask,
3092                                                 MEM_CGROUP_RECLAIM_SOFT);
3093                 nr_reclaimed += reclaimed;
3094                 spin_lock(&mctz->lock);
3095
3096                 /*
3097                  * If we failed to reclaim anything from this memory cgroup
3098                  * it is time to move on to the next cgroup
3099                  */
3100                 next_mz = NULL;
3101                 if (!reclaimed) {
3102                         do {
3103                                 /*
3104                                  * Loop until we find yet another one.
3105                                  *
3106                                  * By the time we get the soft_limit lock
3107                                  * again, someone might have aded the
3108                                  * group back on the RB tree. Iterate to
3109                                  * make sure we get a different mem.
3110                                  * mem_cgroup_largest_soft_limit_node returns
3111                                  * NULL if no other cgroup is present on
3112                                  * the tree
3113                                  */
3114                                 next_mz =
3115                                 __mem_cgroup_largest_soft_limit_node(mctz);
3116                                 if (next_mz == mz) {
3117                                         css_put(&next_mz->mem->css);
3118                                         next_mz = NULL;
3119                                 } else /* next_mz == NULL or other memcg */
3120                                         break;
3121                         } while (1);
3122                 }
3123                 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
3124                 excess = res_counter_soft_limit_excess(&mz->mem->res);
3125                 /*
3126                  * One school of thought says that we should not add
3127                  * back the node to the tree if reclaim returns 0.
3128                  * But our reclaim could return 0, simply because due
3129                  * to priority we are exposing a smaller subset of
3130                  * memory to reclaim from. Consider this as a longer
3131                  * term TODO.
3132                  */
3133                 /* If excess == 0, no tree ops */
3134                 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
3135                 spin_unlock(&mctz->lock);
3136                 css_put(&mz->mem->css);
3137                 loop++;
3138                 /*
3139                  * Could not reclaim anything and there are no more
3140                  * mem cgroups to try or we seem to be looping without
3141                  * reclaiming anything.
3142                  */
3143                 if (!nr_reclaimed &&
3144                         (next_mz == NULL ||
3145                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
3146                         break;
3147         } while (!nr_reclaimed);
3148         if (next_mz)
3149                 css_put(&next_mz->mem->css);
3150         return nr_reclaimed;
3151 }
3152
3153 /*
3154  * This routine traverse page_cgroup in given list and drop them all.
3155  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
3156  */
3157 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
3158                                 int node, int zid, enum lru_list lru)
3159 {
3160         struct zone *zone;
3161         struct mem_cgroup_per_zone *mz;
3162         struct page_cgroup *pc, *busy;
3163         unsigned long flags, loop;
3164         struct list_head *list;
3165         int ret = 0;
3166
3167         zone = &NODE_DATA(node)->node_zones[zid];
3168         mz = mem_cgroup_zoneinfo(mem, node, zid);
3169         list = &mz->lists[lru];
3170
3171         loop = MEM_CGROUP_ZSTAT(mz, lru);
3172         /* give some margin against EBUSY etc...*/
3173         loop += 256;
3174         busy = NULL;
3175         while (loop--) {
3176                 ret = 0;
3177                 spin_lock_irqsave(&zone->lru_lock, flags);
3178                 if (list_empty(list)) {
3179                         spin_unlock_irqrestore(&zone->lru_lock, flags);
3180                         break;
3181                 }
3182                 pc = list_entry(list->prev, struct page_cgroup, lru);
3183                 if (busy == pc) {
3184                         list_move(&pc->lru, list);
3185                         busy = NULL;
3186                         spin_unlock_irqrestore(&zone->lru_lock, flags);
3187                         continue;
3188                 }
3189                 spin_unlock_irqrestore(&zone->lru_lock, flags);
3190
3191                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
3192                 if (ret == -ENOMEM)
3193                         break;
3194
3195                 if (ret == -EBUSY || ret == -EINVAL) {
3196                         /* found lock contention or "pc" is obsolete. */
3197                         busy = pc;
3198                         cond_resched();
3199                 } else
3200                         busy = NULL;
3201         }
3202
3203         if (!ret && !list_empty(list))
3204                 return -EBUSY;
3205         return ret;
3206 }
3207
3208 /*
3209  * make mem_cgroup's charge to be 0 if there is no task.
3210  * This enables deleting this mem_cgroup.
3211  */
3212 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
3213 {
3214         int ret;
3215         int node, zid, shrink;
3216         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
3217         struct cgroup *cgrp = mem->css.cgroup;
3218
3219         css_get(&mem->css);
3220
3221         shrink = 0;
3222         /* should free all ? */
3223         if (free_all)
3224                 goto try_to_free;
3225 move_account:
3226         do {
3227                 ret = -EBUSY;
3228                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
3229                         goto out;
3230                 ret = -EINTR;
3231                 if (signal_pending(current))
3232                         goto out;
3233                 /* This is for making all *used* pages to be on LRU. */
3234                 lru_add_drain_all();
3235                 drain_all_stock_sync();
3236                 ret = 0;
3237                 mem_cgroup_start_move(mem);
3238                 for_each_node_state(node, N_HIGH_MEMORY) {
3239                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
3240                                 enum lru_list l;
3241                                 for_each_lru(l) {
3242                                         ret = mem_cgroup_force_empty_list(mem,
3243                                                         node, zid, l);
3244                                         if (ret)
3245                                                 break;
3246                                 }
3247                         }
3248                         if (ret)
3249                                 break;
3250                 }
3251                 mem_cgroup_end_move(mem);
3252                 memcg_oom_recover(mem);
3253                 /* it seems parent cgroup doesn't have enough mem */
3254                 if (ret == -ENOMEM)
3255                         goto try_to_free;
3256                 cond_resched();
3257         /* "ret" should also be checked to ensure all lists are empty. */
3258         } while (mem->res.usage > 0 || ret);
3259 out:
3260         css_put(&mem->css);
3261         return ret;
3262
3263 try_to_free:
3264         /* returns EBUSY if there is a task or if we come here twice. */
3265         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
3266                 ret = -EBUSY;
3267                 goto out;
3268         }
3269         /* we call try-to-free pages for make this cgroup empty */
3270         lru_add_drain_all();
3271         /* try to free all pages in this cgroup */
3272         shrink = 1;
3273         while (nr_retries && mem->res.usage > 0) {
3274                 int progress;
3275
3276                 if (signal_pending(current)) {
3277                         ret = -EINTR;
3278                         goto out;
3279                 }
3280                 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3281                                                 false, get_swappiness(mem));
3282                 if (!progress) {
3283                         nr_retries--;
3284                         /* maybe some writeback is necessary */
3285                         congestion_wait(BLK_RW_ASYNC, HZ/10);
3286                 }
3287
3288         }
3289         lru_add_drain();
3290         /* try move_account...there may be some *locked* pages. */
3291         goto move_account;
3292 }
3293
3294 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3295 {
3296         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3297 }
3298
3299
3300 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3301 {
3302         return mem_cgroup_from_cont(cont)->use_hierarchy;
3303 }
3304
3305 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3306                                         u64 val)
3307 {
3308         int retval = 0;
3309         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3310         struct cgroup *parent = cont->parent;
3311         struct mem_cgroup *parent_mem = NULL;
3312
3313         if (parent)
3314                 parent_mem = mem_cgroup_from_cont(parent);
3315
3316         cgroup_lock();
3317         /*
3318          * If parent's use_hierarchy is set, we can't make any modifications
3319          * in the child subtrees. If it is unset, then the change can
3320          * occur, provided the current cgroup has no children.
3321          *
3322          * For the root cgroup, parent_mem is NULL, we allow value to be
3323          * set if there are no children.
3324          */
3325         if ((!parent_mem || !parent_mem->use_hierarchy) &&
3326                                 (val == 1 || val == 0)) {
3327                 if (list_empty(&cont->children))
3328                         mem->use_hierarchy = val;
3329                 else
3330                         retval = -EBUSY;
3331         } else
3332                 retval = -EINVAL;
3333         cgroup_unlock();
3334
3335         return retval;
3336 }
3337
3338
3339 static u64 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3340                                 enum mem_cgroup_stat_index idx)
3341 {
3342         struct mem_cgroup *iter;
3343         s64 val = 0;
3344
3345         /* each per cpu's value can be minus.Then, use s64 */
3346         for_each_mem_cgroup_tree(iter, mem)
3347                 val += mem_cgroup_read_stat(iter, idx);
3348
3349         if (val < 0) /* race ? */
3350                 val = 0;
3351         return val;
3352 }
3353
3354 static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3355 {
3356         u64 val;
3357
3358         if (!mem_cgroup_is_root(mem)) {
3359                 if (!swap)
3360                         return res_counter_read_u64(&mem->res, RES_USAGE);
3361                 else
3362                         return res_counter_read_u64(&mem->memsw, RES_USAGE);
3363         }
3364
3365         val = mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE);
3366         val += mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS);
3367
3368         if (swap)
3369                 val += mem_cgroup_get_recursive_idx_stat(mem,
3370                                 MEM_CGROUP_STAT_SWAPOUT);
3371
3372         return val << PAGE_SHIFT;
3373 }
3374
3375 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
3376 {
3377         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3378         u64 val;
3379         int type, name;
3380
3381         type = MEMFILE_TYPE(cft->private);
3382         name = MEMFILE_ATTR(cft->private);
3383         switch (type) {
3384         case _MEM:
3385                 if (name == RES_USAGE)
3386                         val = mem_cgroup_usage(mem, false);
3387                 else
3388                         val = res_counter_read_u64(&mem->res, name);
3389                 break;
3390         case _MEMSWAP:
3391                 if (name == RES_USAGE)
3392                         val = mem_cgroup_usage(mem, true);
3393                 else
3394                         val = res_counter_read_u64(&mem->memsw, name);
3395                 break;
3396         default:
3397                 BUG();
3398                 break;
3399         }
3400         return val;
3401 }
3402 /*
3403  * The user of this function is...
3404  * RES_LIMIT.
3405  */
3406 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3407                             const char *buffer)
3408 {
3409         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3410         int type, name;
3411         unsigned long long val;
3412         int ret;
3413
3414         type = MEMFILE_TYPE(cft->private);
3415         name = MEMFILE_ATTR(cft->private);
3416         switch (name) {
3417         case RES_LIMIT:
3418                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3419                         ret = -EINVAL;
3420                         break;
3421                 }
3422                 /* This function does all necessary parse...reuse it */
3423                 ret = res_counter_memparse_write_strategy(buffer, &val);
3424                 if (ret)
3425                         break;
3426                 if (type == _MEM)
3427                         ret = mem_cgroup_resize_limit(memcg, val);
3428                 else
3429                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
3430                 break;
3431         case RES_SOFT_LIMIT:
3432                 ret = res_counter_memparse_write_strategy(buffer, &val);
3433                 if (ret)
3434                         break;
3435                 /*
3436                  * For memsw, soft limits are hard to implement in terms
3437                  * of semantics, for now, we support soft limits for
3438                  * control without swap
3439                  */
3440                 if (type == _MEM)
3441                         ret = res_counter_set_soft_limit(&memcg->res, val);
3442                 else
3443                         ret = -EINVAL;
3444                 break;
3445         default:
3446                 ret = -EINVAL; /* should be BUG() ? */
3447                 break;
3448         }
3449         return ret;
3450 }
3451
3452 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3453                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3454 {
3455         struct cgroup *cgroup;
3456         unsigned long long min_limit, min_memsw_limit, tmp;
3457
3458         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3459         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3460         cgroup = memcg->css.cgroup;
3461         if (!memcg->use_hierarchy)
3462                 goto out;
3463
3464         while (cgroup->parent) {
3465                 cgroup = cgroup->parent;
3466                 memcg = mem_cgroup_from_cont(cgroup);
3467                 if (!memcg->use_hierarchy)
3468                         break;
3469                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3470                 min_limit = min(min_limit, tmp);
3471                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3472                 min_memsw_limit = min(min_memsw_limit, tmp);
3473         }
3474 out:
3475         *mem_limit = min_limit;
3476         *memsw_limit = min_memsw_limit;
3477         return;
3478 }
3479
3480 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
3481 {
3482         struct mem_cgroup *mem;
3483         int type, name;
3484
3485         mem = mem_cgroup_from_cont(cont);
3486         type = MEMFILE_TYPE(event);
3487         name = MEMFILE_ATTR(event);
3488         switch (name) {
3489         case RES_MAX_USAGE:
3490                 if (type == _MEM)
3491                         res_counter_reset_max(&mem->res);
3492                 else
3493                         res_counter_reset_max(&mem->memsw);
3494                 break;
3495         case RES_FAILCNT:
3496                 if (type == _MEM)
3497                         res_counter_reset_failcnt(&mem->res);
3498                 else
3499                         res_counter_reset_failcnt(&mem->memsw);
3500                 break;
3501         }
3502
3503         return 0;
3504 }
3505
3506 static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3507                                         struct cftype *cft)
3508 {
3509         return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3510 }
3511
3512 #ifdef CONFIG_MMU
3513 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3514                                         struct cftype *cft, u64 val)
3515 {
3516         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3517
3518         if (val >= (1 << NR_MOVE_TYPE))
3519                 return -EINVAL;
3520         /*
3521          * We check this value several times in both in can_attach() and
3522          * attach(), so we need cgroup lock to prevent this value from being
3523          * inconsistent.
3524          */
3525         cgroup_lock();
3526         mem->move_charge_at_immigrate = val;
3527         cgroup_unlock();
3528
3529         return 0;
3530 }
3531 #else
3532 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3533                                         struct cftype *cft, u64 val)
3534 {
3535         return -ENOSYS;
3536 }
3537 #endif
3538
3539
3540 /* For read statistics */
3541 enum {
3542         MCS_CACHE,
3543         MCS_RSS,
3544         MCS_FILE_MAPPED,
3545         MCS_PGPGIN,
3546         MCS_PGPGOUT,
3547         MCS_SWAP,
3548         MCS_INACTIVE_ANON,
3549         MCS_ACTIVE_ANON,
3550         MCS_INACTIVE_FILE,
3551         MCS_ACTIVE_FILE,
3552         MCS_UNEVICTABLE,
3553         NR_MCS_STAT,
3554 };
3555
3556 struct mcs_total_stat {
3557         s64 stat[NR_MCS_STAT];
3558 };
3559
3560 struct {
3561         char *local_name;
3562         char *total_name;
3563 } memcg_stat_strings[NR_MCS_STAT] = {
3564         {"cache", "total_cache"},
3565         {"rss", "total_rss"},
3566         {"mapped_file", "total_mapped_file"},
3567         {"pgpgin", "total_pgpgin"},
3568         {"pgpgout", "total_pgpgout"},
3569         {"swap", "total_swap"},
3570         {"inactive_anon", "total_inactive_anon"},
3571         {"active_anon", "total_active_anon"},
3572         {"inactive_file", "total_inactive_file"},
3573         {"active_file", "total_active_file"},
3574         {"unevictable", "total_unevictable"}
3575 };
3576
3577
3578 static void
3579 mem_cgroup_get_local_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3580 {
3581         s64 val;
3582
3583         /* per cpu stat */
3584         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
3585         s->stat[MCS_CACHE] += val * PAGE_SIZE;
3586         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
3587         s->stat[MCS_RSS] += val * PAGE_SIZE;
3588         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
3589         s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
3590         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
3591         s->stat[MCS_PGPGIN] += val;
3592         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
3593         s->stat[MCS_PGPGOUT] += val;
3594         if (do_swap_account) {
3595                 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
3596                 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3597         }
3598
3599         /* per zone stat */
3600         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3601         s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3602         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3603         s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3604         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3605         s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3606         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3607         s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3608         val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3609         s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
3610 }
3611
3612 static void
3613 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3614 {
3615         struct mem_cgroup *iter;
3616
3617         for_each_mem_cgroup_tree(iter, mem)
3618                 mem_cgroup_get_local_stat(iter, s);
3619 }
3620
3621 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3622                                  struct cgroup_map_cb *cb)
3623 {
3624         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
3625         struct mcs_total_stat mystat;
3626         int i;
3627
3628         memset(&mystat, 0, sizeof(mystat));
3629         mem_cgroup_get_local_stat(mem_cont, &mystat);
3630
3631         for (i = 0; i < NR_MCS_STAT; i++) {
3632                 if (i == MCS_SWAP && !do_swap_account)
3633                         continue;
3634                 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
3635         }
3636
3637         /* Hierarchical information */
3638         {
3639                 unsigned long long limit, memsw_limit;
3640                 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3641                 cb->fill(cb, "hierarchical_memory_limit", limit);
3642                 if (do_swap_account)
3643                         cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3644         }
3645
3646         memset(&mystat, 0, sizeof(mystat));
3647         mem_cgroup_get_total_stat(mem_cont, &mystat);
3648         for (i = 0; i < NR_MCS_STAT; i++) {
3649                 if (i == MCS_SWAP && !do_swap_account)
3650                         continue;
3651                 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
3652         }
3653
3654 #ifdef CONFIG_DEBUG_VM
3655         cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
3656
3657         {
3658                 int nid, zid;
3659                 struct mem_cgroup_per_zone *mz;
3660                 unsigned long recent_rotated[2] = {0, 0};
3661                 unsigned long recent_scanned[2] = {0, 0};
3662
3663                 for_each_online_node(nid)
3664                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3665                                 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3666
3667                                 recent_rotated[0] +=
3668                                         mz->reclaim_stat.recent_rotated[0];
3669                                 recent_rotated[1] +=
3670                                         mz->reclaim_stat.recent_rotated[1];
3671                                 recent_scanned[0] +=
3672                                         mz->reclaim_stat.recent_scanned[0];
3673                                 recent_scanned[1] +=
3674                                         mz->reclaim_stat.recent_scanned[1];
3675                         }
3676                 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3677                 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3678                 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3679                 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3680         }
3681 #endif
3682
3683         return 0;
3684 }
3685
3686 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3687 {
3688         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3689
3690         return get_swappiness(memcg);
3691 }
3692
3693 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3694                                        u64 val)
3695 {
3696         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3697         struct mem_cgroup *parent;
3698
3699         if (val > 100)
3700                 return -EINVAL;
3701
3702         if (cgrp->parent == NULL)
3703                 return -EINVAL;
3704
3705         parent = mem_cgroup_from_cont(cgrp->parent);
3706
3707         cgroup_lock();
3708
3709         /* If under hierarchy, only empty-root can set this value */
3710         if ((parent->use_hierarchy) ||
3711             (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3712                 cgroup_unlock();
3713                 return -EINVAL;
3714         }
3715
3716         spin_lock(&memcg->reclaim_param_lock);
3717         memcg->swappiness = val;
3718         spin_unlock(&memcg->reclaim_param_lock);
3719
3720         cgroup_unlock();
3721
3722         return 0;
3723 }
3724
3725 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3726 {
3727         struct mem_cgroup_threshold_ary *t;
3728         u64 usage;
3729         int i;
3730
3731         rcu_read_lock();
3732         if (!swap)
3733                 t = rcu_dereference(memcg->thresholds.primary);
3734         else
3735                 t = rcu_dereference(memcg->memsw_thresholds.primary);
3736
3737         if (!t)
3738                 goto unlock;
3739
3740         usage = mem_cgroup_usage(memcg, swap);
3741
3742         /*
3743          * current_threshold points to threshold just below usage.
3744          * If it's not true, a threshold was crossed after last
3745          * call of __mem_cgroup_threshold().
3746          */
3747         i = t->current_threshold;
3748
3749         /*
3750          * Iterate backward over array of thresholds starting from
3751          * current_threshold and check if a threshold is crossed.
3752          * If none of thresholds below usage is crossed, we read
3753          * only one element of the array here.
3754          */
3755         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3756                 eventfd_signal(t->entries[i].eventfd, 1);
3757
3758         /* i = current_threshold + 1 */
3759         i++;
3760
3761         /*
3762          * Iterate forward over array of thresholds starting from
3763          * current_threshold+1 and check if a threshold is crossed.
3764          * If none of thresholds above usage is crossed, we read
3765          * only one element of the array here.
3766          */
3767         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3768                 eventfd_signal(t->entries[i].eventfd, 1);
3769
3770         /* Update current_threshold */
3771         t->current_threshold = i - 1;
3772 unlock:
3773         rcu_read_unlock();
3774 }
3775
3776 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3777 {
3778         while (memcg) {
3779                 __mem_cgroup_threshold(memcg, false);
3780                 if (do_swap_account)
3781                         __mem_cgroup_threshold(memcg, true);
3782
3783                 memcg = parent_mem_cgroup(memcg);
3784         }
3785 }
3786
3787 static int compare_thresholds(const void *a, const void *b)
3788 {
3789         const struct mem_cgroup_threshold *_a = a;
3790         const struct mem_cgroup_threshold *_b = b;
3791
3792         return _a->threshold - _b->threshold;
3793 }
3794
3795 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem)
3796 {
3797         struct mem_cgroup_eventfd_list *ev;
3798
3799         list_for_each_entry(ev, &mem->oom_notify, list)
3800                 eventfd_signal(ev->eventfd, 1);
3801         return 0;
3802 }
3803
3804 static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3805 {
3806         struct mem_cgroup *iter;
3807
3808         for_each_mem_cgroup_tree(iter, mem)
3809                 mem_cgroup_oom_notify_cb(iter);
3810 }
3811
3812 static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3813         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3814 {
3815         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3816         struct mem_cgroup_thresholds *thresholds;
3817         struct mem_cgroup_threshold_ary *new;
3818         int type = MEMFILE_TYPE(cft->private);
3819         u64 threshold, usage;
3820         int i, size, ret;
3821
3822         ret = res_counter_memparse_write_strategy(args, &threshold);
3823         if (ret)
3824                 return ret;
3825
3826         mutex_lock(&memcg->thresholds_lock);
3827
3828         if (type == _MEM)
3829                 thresholds = &memcg->thresholds;
3830         else if (type == _MEMSWAP)
3831                 thresholds = &memcg->memsw_thresholds;
3832         else
3833                 BUG();
3834
3835         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3836
3837         /* Check if a threshold crossed before adding a new one */
3838         if (thresholds->primary)
3839                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3840
3841         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3842
3843         /* Allocate memory for new array of thresholds */
3844         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3845                         GFP_KERNEL);
3846         if (!new) {
3847                 ret = -ENOMEM;
3848                 goto unlock;
3849         }
3850         new->size = size;
3851
3852         /* Copy thresholds (if any) to new array */
3853         if (thresholds->primary) {
3854                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3855                                 sizeof(struct mem_cgroup_threshold));
3856         }
3857
3858         /* Add new threshold */
3859         new->entries[size - 1].eventfd = eventfd;
3860         new->entries[size - 1].threshold = threshold;
3861
3862         /* Sort thresholds. Registering of new threshold isn't time-critical */
3863         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3864                         compare_thresholds, NULL);
3865
3866         /* Find current threshold */
3867         new->current_threshold = -1;
3868         for (i = 0; i < size; i++) {
3869                 if (new->entries[i].threshold < usage) {
3870                         /*
3871                          * new->current_threshold will not be used until
3872                          * rcu_assign_pointer(), so it's safe to increment
3873                          * it here.
3874                          */
3875                         ++new->current_threshold;
3876                 }
3877         }
3878
3879         /* Free old spare buffer and save old primary buffer as spare */
3880         kfree(thresholds->spare);
3881         thresholds->spare = thresholds->primary;
3882
3883         rcu_assign_pointer(thresholds->primary, new);
3884
3885         /* To be sure that nobody uses thresholds */
3886         synchronize_rcu();
3887
3888 unlock:
3889         mutex_unlock(&memcg->thresholds_lock);
3890
3891         return ret;
3892 }
3893
3894 static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
3895         struct cftype *cft, struct eventfd_ctx *eventfd)
3896 {
3897         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3898         struct mem_cgroup_thresholds *thresholds;
3899         struct mem_cgroup_threshold_ary *new;
3900         int type = MEMFILE_TYPE(cft->private);
3901         u64 usage;
3902         int i, j, size;
3903
3904         mutex_lock(&memcg->thresholds_lock);
3905         if (type == _MEM)
3906                 thresholds = &memcg->thresholds;
3907         else if (type == _MEMSWAP)
3908                 thresholds = &memcg->memsw_thresholds;
3909         else
3910                 BUG();
3911
3912         /*
3913          * Something went wrong if we trying to unregister a threshold
3914          * if we don't have thresholds
3915          */
3916         BUG_ON(!thresholds);
3917
3918         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3919
3920         /* Check if a threshold crossed before removing */
3921         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3922
3923         /* Calculate new number of threshold */
3924         size = 0;
3925         for (i = 0; i < thresholds->primary->size; i++) {
3926                 if (thresholds->primary->entries[i].eventfd != eventfd)
3927                         size++;
3928         }
3929
3930         new = thresholds->spare;
3931
3932         /* Set thresholds array to NULL if we don't have thresholds */
3933         if (!size) {
3934                 kfree(new);
3935                 new = NULL;
3936                 goto swap_buffers;
3937         }
3938
3939         new->size = size;
3940
3941         /* Copy thresholds and find current threshold */
3942         new->current_threshold = -1;
3943         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3944                 if (thresholds->primary->entries[i].eventfd == eventfd)
3945                         continue;
3946
3947                 new->entries[j] = thresholds->primary->entries[i];
3948                 if (new->entries[j].threshold < usage) {
3949                         /*
3950                          * new->current_threshold will not be used
3951                          * until rcu_assign_pointer(), so it's safe to increment
3952                          * it here.
3953                          */
3954                         ++new->current_threshold;
3955                 }
3956                 j++;
3957         }
3958
3959 swap_buffers:
3960         /* Swap primary and spare array */
3961         thresholds->spare = thresholds->primary;
3962         rcu_assign_pointer(thresholds->primary, new);
3963
3964         /* To be sure that nobody uses thresholds */
3965         synchronize_rcu();
3966
3967         mutex_unlock(&memcg->thresholds_lock);
3968 }
3969
3970 static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
3971         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3972 {
3973         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3974         struct mem_cgroup_eventfd_list *event;
3975         int type = MEMFILE_TYPE(cft->private);
3976
3977         BUG_ON(type != _OOM_TYPE);
3978         event = kmalloc(sizeof(*event), GFP_KERNEL);
3979         if (!event)
3980                 return -ENOMEM;
3981
3982         mutex_lock(&memcg_oom_mutex);
3983
3984         event->eventfd = eventfd;
3985         list_add(&event->list, &memcg->oom_notify);
3986
3987         /* already in OOM ? */
3988         if (atomic_read(&memcg->oom_lock))
3989                 eventfd_signal(eventfd, 1);
3990         mutex_unlock(&memcg_oom_mutex);
3991
3992         return 0;
3993 }
3994
3995 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
3996         struct cftype *cft, struct eventfd_ctx *eventfd)
3997 {
3998         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3999         struct mem_cgroup_eventfd_list *ev, *tmp;
4000         int type = MEMFILE_TYPE(cft->private);
4001
4002         BUG_ON(type != _OOM_TYPE);
4003
4004         mutex_lock(&memcg_oom_mutex);
4005
4006         list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
4007                 if (ev->eventfd == eventfd) {
4008                         list_del(&ev->list);
4009                         kfree(ev);
4010                 }
4011         }
4012
4013         mutex_unlock(&memcg_oom_mutex);
4014 }
4015
4016 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
4017         struct cftype *cft,  struct cgroup_map_cb *cb)
4018 {
4019         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4020
4021         cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
4022
4023         if (atomic_read(&mem->oom_lock))
4024                 cb->fill(cb, "under_oom", 1);
4025         else
4026                 cb->fill(cb, "under_oom", 0);
4027         return 0;
4028 }
4029
4030 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
4031         struct cftype *cft, u64 val)
4032 {
4033         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
4034         struct mem_cgroup *parent;
4035
4036         /* cannot set to root cgroup and only 0 and 1 are allowed */
4037         if (!cgrp->parent || !((val == 0) || (val == 1)))
4038                 return -EINVAL;
4039
4040         parent = mem_cgroup_from_cont(cgrp->parent);
4041
4042         cgroup_lock();
4043         /* oom-kill-disable is a flag for subhierarchy. */
4044         if ((parent->use_hierarchy) ||
4045             (mem->use_hierarchy && !list_empty(&cgrp->children))) {
4046                 cgroup_unlock();
4047                 return -EINVAL;
4048         }
4049         mem->oom_kill_disable = val;
4050         if (!val)
4051                 memcg_oom_recover(mem);
4052         cgroup_unlock();
4053         return 0;
4054 }
4055
4056 static struct cftype mem_cgroup_files[] = {
4057         {
4058                 .name = "usage_in_bytes",
4059                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
4060                 .read_u64 = mem_cgroup_read,
4061                 .register_event = mem_cgroup_usage_register_event,
4062                 .unregister_event = mem_cgroup_usage_unregister_event,
4063         },
4064         {
4065                 .name = "max_usage_in_bytes",
4066                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
4067                 .trigger = mem_cgroup_reset,
4068                 .read_u64 = mem_cgroup_read,
4069         },
4070         {
4071                 .name = "limit_in_bytes",
4072                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
4073                 .write_string = mem_cgroup_write,
4074                 .read_u64 = mem_cgroup_read,
4075         },
4076         {
4077                 .name = "soft_limit_in_bytes",
4078                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
4079                 .write_string = mem_cgroup_write,
4080                 .read_u64 = mem_cgroup_read,
4081         },
4082         {
4083                 .name = "failcnt",
4084                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
4085                 .trigger = mem_cgroup_reset,
4086                 .read_u64 = mem_cgroup_read,
4087         },
4088         {
4089                 .name = "stat",
4090                 .read_map = mem_control_stat_show,
4091         },
4092         {
4093                 .name = "force_empty",
4094                 .trigger = mem_cgroup_force_empty_write,
4095         },
4096         {
4097                 .name = "use_hierarchy",
4098                 .write_u64 = mem_cgroup_hierarchy_write,
4099                 .read_u64 = mem_cgroup_hierarchy_read,
4100         },
4101         {
4102                 .name = "swappiness",
4103                 .read_u64 = mem_cgroup_swappiness_read,
4104                 .write_u64 = mem_cgroup_swappiness_write,
4105         },
4106         {
4107                 .name = "move_charge_at_immigrate",
4108                 .read_u64 = mem_cgroup_move_charge_read,
4109                 .write_u64 = mem_cgroup_move_charge_write,
4110         },
4111         {
4112                 .name = "oom_control",
4113                 .read_map = mem_cgroup_oom_control_read,
4114                 .write_u64 = mem_cgroup_oom_control_write,
4115                 .register_event = mem_cgroup_oom_register_event,
4116                 .unregister_event = mem_cgroup_oom_unregister_event,
4117                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
4118         },
4119 };
4120
4121 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4122 static struct cftype memsw_cgroup_files[] = {
4123         {
4124                 .name = "memsw.usage_in_bytes",
4125                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
4126                 .read_u64 = mem_cgroup_read,
4127                 .register_event = mem_cgroup_usage_register_event,
4128                 .unregister_event = mem_cgroup_usage_unregister_event,
4129         },
4130         {
4131                 .name = "memsw.max_usage_in_bytes",
4132                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
4133                 .trigger = mem_cgroup_reset,
4134                 .read_u64 = mem_cgroup_read,
4135         },
4136         {
4137                 .name = "memsw.limit_in_bytes",
4138                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
4139                 .write_string = mem_cgroup_write,
4140                 .read_u64 = mem_cgroup_read,
4141         },
4142         {
4143                 .name = "memsw.failcnt",
4144                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
4145                 .trigger = mem_cgroup_reset,
4146                 .read_u64 = mem_cgroup_read,
4147         },
4148 };
4149
4150 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4151 {
4152         if (!do_swap_account)
4153                 return 0;
4154         return cgroup_add_files(cont, ss, memsw_cgroup_files,
4155                                 ARRAY_SIZE(memsw_cgroup_files));
4156 };
4157 #else
4158 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
4159 {
4160         return 0;
4161 }
4162 #endif
4163
4164 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4165 {
4166         struct mem_cgroup_per_node *pn;
4167         struct mem_cgroup_per_zone *mz;
4168         enum lru_list l;
4169         int zone, tmp = node;
4170         /*
4171          * This routine is called against possible nodes.
4172          * But it's BUG to call kmalloc() against offline node.
4173          *
4174          * TODO: this routine can waste much memory for nodes which will
4175          *       never be onlined. It's better to use memory hotplug callback
4176          *       function.
4177          */
4178         if (!node_state(node, N_NORMAL_MEMORY))
4179                 tmp = -1;
4180         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
4181         if (!pn)
4182                 return 1;
4183
4184         mem->info.nodeinfo[node] = pn;
4185         memset(pn, 0, sizeof(*pn));
4186
4187         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4188                 mz = &pn->zoneinfo[zone];
4189                 for_each_lru(l)
4190                         INIT_LIST_HEAD(&mz->lists[l]);
4191                 mz->usage_in_excess = 0;
4192                 mz->on_tree = false;
4193                 mz->mem = mem;
4194         }
4195         return 0;
4196 }
4197
4198 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
4199 {
4200         kfree(mem->info.nodeinfo[node]);
4201 }
4202
4203 static struct mem_cgroup *mem_cgroup_alloc(void)
4204 {
4205         struct mem_cgroup *mem;
4206         int size = sizeof(struct mem_cgroup);
4207
4208         /* Can be very big if MAX_NUMNODES is very big */
4209         if (size < PAGE_SIZE)
4210                 mem = kmalloc(size, GFP_KERNEL);
4211         else
4212                 mem = vmalloc(size);
4213
4214         if (!mem)
4215                 return NULL;
4216
4217         memset(mem, 0, size);
4218         mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
4219         if (!mem->stat)
4220                 goto out_free;
4221         spin_lock_init(&mem->pcp_counter_lock);
4222         return mem;
4223
4224 out_free:
4225         if (size < PAGE_SIZE)
4226                 kfree(mem);
4227         else
4228                 vfree(mem);
4229         return NULL;
4230 }
4231
4232 /*
4233  * At destroying mem_cgroup, references from swap_cgroup can remain.
4234  * (scanning all at force_empty is too costly...)
4235  *
4236  * Instead of clearing all references at force_empty, we remember
4237  * the number of reference from swap_cgroup and free mem_cgroup when
4238  * it goes down to 0.
4239  *
4240  * Removal of cgroup itself succeeds regardless of refs from swap.
4241  */
4242
4243 static void __mem_cgroup_free(struct mem_cgroup *mem)
4244 {
4245         int node;
4246
4247         mem_cgroup_remove_from_trees(mem);
4248         free_css_id(&mem_cgroup_subsys, &mem->css);
4249
4250         for_each_node_state(node, N_POSSIBLE)
4251                 free_mem_cgroup_per_zone_info(mem, node);
4252
4253         free_percpu(mem->stat);
4254         if (sizeof(struct mem_cgroup) < PAGE_SIZE)
4255                 kfree(mem);
4256         else
4257                 vfree(mem);
4258 }
4259
4260 static void mem_cgroup_get(struct mem_cgroup *mem)
4261 {
4262         atomic_inc(&mem->refcnt);
4263 }
4264
4265 static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
4266 {
4267         if (atomic_sub_and_test(count, &mem->refcnt)) {
4268                 struct mem_cgroup *parent = parent_mem_cgroup(mem);
4269                 __mem_cgroup_free(mem);
4270                 if (parent)
4271                         mem_cgroup_put(parent);
4272         }
4273 }
4274
4275 static void mem_cgroup_put(struct mem_cgroup *mem)
4276 {
4277         __mem_cgroup_put(mem, 1);
4278 }
4279
4280 /*
4281  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4282  */
4283 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4284 {
4285         if (!mem->res.parent)
4286                 return NULL;
4287         return mem_cgroup_from_res_counter(mem->res.parent, res);
4288 }
4289
4290 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4291 static void __init enable_swap_cgroup(void)
4292 {
4293         if (!mem_cgroup_disabled() && really_do_swap_account)
4294                 do_swap_account = 1;
4295 }
4296 #else
4297 static void __init enable_swap_cgroup(void)
4298 {
4299 }
4300 #endif
4301
4302 static int mem_cgroup_soft_limit_tree_init(void)
4303 {
4304         struct mem_cgroup_tree_per_node *rtpn;
4305         struct mem_cgroup_tree_per_zone *rtpz;
4306         int tmp, node, zone;
4307
4308         for_each_node_state(node, N_POSSIBLE) {
4309                 tmp = node;
4310                 if (!node_state(node, N_NORMAL_MEMORY))
4311                         tmp = -1;
4312                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4313                 if (!rtpn)
4314                         return 1;
4315
4316                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4317
4318                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4319                         rtpz = &rtpn->rb_tree_per_zone[zone];
4320                         rtpz->rb_root = RB_ROOT;
4321                         spin_lock_init(&rtpz->lock);
4322                 }
4323         }
4324         return 0;
4325 }
4326
4327 static struct cgroup_subsys_state * __ref
4328 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4329 {
4330         struct mem_cgroup *mem, *parent;
4331         long error = -ENOMEM;
4332         int node;
4333
4334         mem = mem_cgroup_alloc();
4335         if (!mem)
4336                 return ERR_PTR(error);
4337
4338         for_each_node_state(node, N_POSSIBLE)
4339                 if (alloc_mem_cgroup_per_zone_info(mem, node))
4340                         goto free_out;
4341
4342         /* root ? */
4343         if (cont->parent == NULL) {
4344                 int cpu;
4345                 enable_swap_cgroup();
4346                 parent = NULL;
4347                 root_mem_cgroup = mem;
4348                 if (mem_cgroup_soft_limit_tree_init())
4349                         goto free_out;
4350                 for_each_possible_cpu(cpu) {
4351                         struct memcg_stock_pcp *stock =
4352                                                 &per_cpu(memcg_stock, cpu);
4353                         INIT_WORK(&stock->work, drain_local_stock);
4354                 }
4355                 hotcpu_notifier(memcg_cpu_hotplug_callback, 0);
4356         } else {
4357                 parent = mem_cgroup_from_cont(cont->parent);
4358                 mem->use_hierarchy = parent->use_hierarchy;
4359                 mem->oom_kill_disable = parent->oom_kill_disable;
4360         }
4361
4362         if (parent && parent->use_hierarchy) {
4363                 res_counter_init(&mem->res, &parent->res);
4364                 res_counter_init(&mem->memsw, &parent->memsw);
4365                 /*
4366                  * We increment refcnt of the parent to ensure that we can
4367                  * safely access it on res_counter_charge/uncharge.
4368                  * This refcnt will be decremented when freeing this
4369                  * mem_cgroup(see mem_cgroup_put).
4370                  */
4371                 mem_cgroup_get(parent);
4372         } else {
4373                 res_counter_init(&mem->res, NULL);
4374                 res_counter_init(&mem->memsw, NULL);
4375         }
4376         mem->last_scanned_child = 0;
4377         spin_lock_init(&mem->reclaim_param_lock);
4378         INIT_LIST_HEAD(&mem->oom_notify);
4379
4380         if (parent)
4381                 mem->swappiness = get_swappiness(parent);
4382         atomic_set(&mem->refcnt, 1);
4383         mem->move_charge_at_immigrate = 0;
4384         mutex_init(&mem->thresholds_lock);
4385         return &mem->css;
4386 free_out:
4387         __mem_cgroup_free(mem);
4388         root_mem_cgroup = NULL;
4389         return ERR_PTR(error);
4390 }
4391
4392 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
4393                                         struct cgroup *cont)
4394 {
4395         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4396
4397         return mem_cgroup_force_empty(mem, false);
4398 }
4399
4400 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4401                                 struct cgroup *cont)
4402 {
4403         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4404
4405         mem_cgroup_put(mem);
4406 }
4407
4408 static int mem_cgroup_populate(struct cgroup_subsys *ss,
4409                                 struct cgroup *cont)
4410 {
4411         int ret;
4412
4413         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4414                                 ARRAY_SIZE(mem_cgroup_files));
4415
4416         if (!ret)
4417                 ret = register_memsw_files(cont, ss);
4418         return ret;
4419 }
4420
4421 #ifdef CONFIG_MMU
4422 /* Handlers for move charge at task migration. */
4423 #define PRECHARGE_COUNT_AT_ONCE 256
4424 static int mem_cgroup_do_precharge(unsigned long count)
4425 {
4426         int ret = 0;
4427         int batch_count = PRECHARGE_COUNT_AT_ONCE;
4428         struct mem_cgroup *mem = mc.to;
4429
4430         if (mem_cgroup_is_root(mem)) {
4431                 mc.precharge += count;
4432                 /* we don't need css_get for root */
4433                 return ret;
4434         }
4435         /* try to charge at once */
4436         if (count > 1) {
4437                 struct res_counter *dummy;
4438                 /*
4439                  * "mem" cannot be under rmdir() because we've already checked
4440                  * by cgroup_lock_live_cgroup() that it is not removed and we
4441                  * are still under the same cgroup_mutex. So we can postpone
4442                  * css_get().
4443                  */
4444                 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4445                         goto one_by_one;
4446                 if (do_swap_account && res_counter_charge(&mem->memsw,
4447                                                 PAGE_SIZE * count, &dummy)) {
4448                         res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4449                         goto one_by_one;
4450                 }
4451                 mc.precharge += count;
4452                 return ret;
4453         }
4454 one_by_one:
4455         /* fall back to one by one charge */
4456         while (count--) {
4457                 if (signal_pending(current)) {
4458                         ret = -EINTR;
4459                         break;
4460                 }
4461                 if (!batch_count--) {
4462                         batch_count = PRECHARGE_COUNT_AT_ONCE;
4463                         cond_resched();
4464                 }
4465                 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
4466                 if (ret || !mem)
4467                         /* mem_cgroup_clear_mc() will do uncharge later */
4468                         return -ENOMEM;
4469                 mc.precharge++;
4470         }
4471         return ret;
4472 }
4473
4474 /**
4475  * is_target_pte_for_mc - check a pte whether it is valid for move charge
4476  * @vma: the vma the pte to be checked belongs
4477  * @addr: the address corresponding to the pte to be checked
4478  * @ptent: the pte to be checked
4479  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4480  *
4481  * Returns
4482  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4483  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4484  *     move charge. if @target is not NULL, the page is stored in target->page
4485  *     with extra refcnt got(Callers should handle it).
4486  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4487  *     target for charge migration. if @target is not NULL, the entry is stored
4488  *     in target->ent.
4489  *
4490  * Called with pte lock held.
4491  */
4492 union mc_target {
4493         struct page     *page;
4494         swp_entry_t     ent;
4495 };
4496
4497 enum mc_target_type {
4498         MC_TARGET_NONE, /* not used */
4499         MC_TARGET_PAGE,
4500         MC_TARGET_SWAP,
4501 };
4502
4503 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4504                                                 unsigned long addr, pte_t ptent)
4505 {
4506         struct page *page = vm_normal_page(vma, addr, ptent);
4507
4508         if (!page || !page_mapped(page))
4509                 return NULL;
4510         if (PageAnon(page)) {
4511                 /* we don't move shared anon */
4512                 if (!move_anon() || page_mapcount(page) > 2)
4513                         return NULL;
4514         } else if (!move_file())
4515                 /* we ignore mapcount for file pages */
4516                 return NULL;
4517         if (!get_page_unless_zero(page))
4518                 return NULL;
4519
4520         return page;
4521 }
4522
4523 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4524                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4525 {
4526         int usage_count;
4527         struct page *page = NULL;
4528         swp_entry_t ent = pte_to_swp_entry(ptent);
4529
4530         if (!move_anon() || non_swap_entry(ent))
4531                 return NULL;
4532         usage_count = mem_cgroup_count_swap_user(ent, &page);
4533         if (usage_count > 1) { /* we don't move shared anon */
4534                 if (page)
4535                         put_page(page);
4536                 return NULL;
4537         }
4538         if (do_swap_account)
4539                 entry->val = ent.val;
4540
4541         return page;
4542 }
4543
4544 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4545                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4546 {
4547         struct page *page = NULL;
4548         struct inode *inode;
4549         struct address_space *mapping;
4550         pgoff_t pgoff;
4551
4552         if (!vma->vm_file) /* anonymous vma */
4553                 return NULL;
4554         if (!move_file())
4555                 return NULL;
4556
4557         inode = vma->vm_file->f_path.dentry->d_inode;
4558         mapping = vma->vm_file->f_mapping;
4559         if (pte_none(ptent))
4560                 pgoff = linear_page_index(vma, addr);
4561         else /* pte_file(ptent) is true */
4562                 pgoff = pte_to_pgoff(ptent);
4563
4564         /* page is moved even if it's not RSS of this task(page-faulted). */
4565         if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4566                 page = find_get_page(mapping, pgoff);
4567         } else { /* shmem/tmpfs file. we should take account of swap too. */
4568                 swp_entry_t ent;
4569                 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4570                 if (do_swap_account)
4571                         entry->val = ent.val;
4572         }
4573
4574         return page;
4575 }
4576
4577 static int is_target_pte_for_mc(struct vm_area_struct *vma,
4578                 unsigned long addr, pte_t ptent, union mc_target *target)
4579 {
4580         struct page *page = NULL;
4581         struct page_cgroup *pc;
4582         int ret = 0;
4583         swp_entry_t ent = { .val = 0 };
4584
4585         if (pte_present(ptent))
4586                 page = mc_handle_present_pte(vma, addr, ptent);
4587         else if (is_swap_pte(ptent))
4588                 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4589         else if (pte_none(ptent) || pte_file(ptent))
4590                 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4591
4592         if (!page && !ent.val)
4593                 return 0;
4594         if (page) {
4595                 pc = lookup_page_cgroup(page);
4596                 /*
4597                  * Do only loose check w/o page_cgroup lock.
4598                  * mem_cgroup_move_account() checks the pc is valid or not under
4599                  * the lock.
4600                  */
4601                 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4602                         ret = MC_TARGET_PAGE;
4603                         if (target)
4604                                 target->page = page;
4605                 }
4606                 if (!ret || !target)
4607                         put_page(page);
4608         }
4609         /* There is a swap entry and a page doesn't exist or isn't charged */
4610         if (ent.val && !ret &&
4611                         css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4612                 ret = MC_TARGET_SWAP;
4613                 if (target)
4614                         target->ent = ent;
4615         }
4616         return ret;
4617 }
4618
4619 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4620                                         unsigned long addr, unsigned long end,
4621                                         struct mm_walk *walk)
4622 {
4623         struct vm_area_struct *vma = walk->private;
4624         pte_t *pte;
4625         spinlock_t *ptl;
4626
4627         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4628         for (; addr != end; pte++, addr += PAGE_SIZE)
4629                 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4630                         mc.precharge++; /* increment precharge temporarily */
4631         pte_unmap_unlock(pte - 1, ptl);
4632         cond_resched();
4633
4634         return 0;
4635 }
4636
4637 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4638 {
4639         unsigned long precharge;
4640         struct vm_area_struct *vma;
4641
4642         /* We've already held the mmap_sem */
4643         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4644                 struct mm_walk mem_cgroup_count_precharge_walk = {
4645                         .pmd_entry = mem_cgroup_count_precharge_pte_range,
4646                         .mm = mm,
4647                         .private = vma,
4648                 };
4649                 if (is_vm_hugetlb_page(vma))
4650                         continue;
4651                 walk_page_range(vma->vm_start, vma->vm_end,
4652                                         &mem_cgroup_count_precharge_walk);
4653         }
4654
4655         precharge = mc.precharge;
4656         mc.precharge = 0;
4657
4658         return precharge;
4659 }
4660
4661 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4662 {
4663         return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
4664 }
4665
4666 static void mem_cgroup_clear_mc(void)
4667 {
4668         struct mem_cgroup *from = mc.from;
4669         struct mem_cgroup *to = mc.to;
4670
4671         /* we must uncharge all the leftover precharges from mc.to */
4672         if (mc.precharge) {
4673                 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4674                 mc.precharge = 0;
4675         }
4676         /*
4677          * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4678          * we must uncharge here.
4679          */
4680         if (mc.moved_charge) {
4681                 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4682                 mc.moved_charge = 0;
4683         }
4684         /* we must fixup refcnts and charges */
4685         if (mc.moved_swap) {
4686                 /* uncharge swap account from the old cgroup */
4687                 if (!mem_cgroup_is_root(mc.from))
4688                         res_counter_uncharge(&mc.from->memsw,
4689                                                 PAGE_SIZE * mc.moved_swap);
4690                 __mem_cgroup_put(mc.from, mc.moved_swap);
4691
4692                 if (!mem_cgroup_is_root(mc.to)) {
4693                         /*
4694                          * we charged both to->res and to->memsw, so we should
4695                          * uncharge to->res.
4696                          */
4697                         res_counter_uncharge(&mc.to->res,
4698                                                 PAGE_SIZE * mc.moved_swap);
4699                 }
4700                 /* we've already done mem_cgroup_get(mc.to) */
4701
4702                 mc.moved_swap = 0;
4703         }
4704         if (mc.mm) {
4705                 up_read(&mc.mm->mmap_sem);
4706                 mmput(mc.mm);
4707         }
4708         spin_lock(&mc.lock);
4709         mc.from = NULL;
4710         mc.to = NULL;
4711         spin_unlock(&mc.lock);
4712         mc.moving_task = NULL;
4713         mc.mm = NULL;
4714         mem_cgroup_end_move(from);
4715         memcg_oom_recover(from);
4716         memcg_oom_recover(to);
4717         wake_up_all(&mc.waitq);
4718 }
4719
4720 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4721                                 struct cgroup *cgroup,
4722                                 struct task_struct *p,
4723                                 bool threadgroup)
4724 {
4725         int ret = 0;
4726         struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4727
4728         if (mem->move_charge_at_immigrate) {
4729                 struct mm_struct *mm;
4730                 struct mem_cgroup *from = mem_cgroup_from_task(p);
4731
4732                 VM_BUG_ON(from == mem);
4733
4734                 mm = get_task_mm(p);
4735                 if (!mm)
4736                         return 0;
4737                 /* We move charges only when we move a owner of the mm */
4738                 if (mm->owner == p) {
4739                         /*
4740                          * We do all the move charge works under one mmap_sem to
4741                          * avoid deadlock with down_write(&mmap_sem)
4742                          * -> try_charge() -> if (mc.moving_task) -> sleep.
4743                          */
4744                         down_read(&mm->mmap_sem);
4745
4746                         VM_BUG_ON(mc.from);
4747                         VM_BUG_ON(mc.to);
4748                         VM_BUG_ON(mc.precharge);
4749                         VM_BUG_ON(mc.moved_charge);
4750                         VM_BUG_ON(mc.moved_swap);
4751                         VM_BUG_ON(mc.moving_task);
4752                         VM_BUG_ON(mc.mm);
4753
4754                         mem_cgroup_start_move(from);
4755                         spin_lock(&mc.lock);
4756                         mc.from = from;
4757                         mc.to = mem;
4758                         mc.precharge = 0;
4759                         mc.moved_charge = 0;
4760                         mc.moved_swap = 0;
4761                         spin_unlock(&mc.lock);
4762                         mc.moving_task = current;
4763                         mc.mm = mm;
4764
4765                         ret = mem_cgroup_precharge_mc(mm);
4766                         if (ret)
4767                                 mem_cgroup_clear_mc();
4768                         /* We call up_read() and mmput() in clear_mc(). */
4769                 } else
4770                         mmput(mm);
4771         }
4772         return ret;
4773 }
4774
4775 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4776                                 struct cgroup *cgroup,
4777                                 struct task_struct *p,
4778                                 bool threadgroup)
4779 {
4780         mem_cgroup_clear_mc();
4781 }
4782
4783 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4784                                 unsigned long addr, unsigned long end,
4785                                 struct mm_walk *walk)
4786 {
4787         int ret = 0;
4788         struct vm_area_struct *vma = walk->private;
4789         pte_t *pte;
4790         spinlock_t *ptl;
4791
4792 retry:
4793         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4794         for (; addr != end; addr += PAGE_SIZE) {
4795                 pte_t ptent = *(pte++);
4796                 union mc_target target;
4797                 int type;
4798                 struct page *page;
4799                 struct page_cgroup *pc;
4800                 swp_entry_t ent;
4801
4802                 if (!mc.precharge)
4803                         break;
4804
4805                 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4806                 switch (type) {
4807                 case MC_TARGET_PAGE:
4808                         page = target.page;
4809                         if (isolate_lru_page(page))
4810                                 goto put;
4811                         pc = lookup_page_cgroup(page);
4812                         if (!mem_cgroup_move_account(pc,
4813                                                 mc.from, mc.to, false)) {
4814                                 mc.precharge--;
4815                                 /* we uncharge from mc.from later. */
4816                                 mc.moved_charge++;
4817                         }
4818                         putback_lru_page(page);
4819 put:                    /* is_target_pte_for_mc() gets the page */
4820                         put_page(page);
4821                         break;
4822                 case MC_TARGET_SWAP:
4823                         ent = target.ent;
4824                         if (!mem_cgroup_move_swap_account(ent,
4825                                                 mc.from, mc.to, false)) {
4826                                 mc.precharge--;
4827                                 /* we fixup refcnts and charges later. */
4828                                 mc.moved_swap++;
4829                         }
4830                         break;
4831                 default:
4832                         break;
4833                 }
4834         }
4835         pte_unmap_unlock(pte - 1, ptl);
4836         cond_resched();
4837
4838         if (addr != end) {
4839                 /*
4840                  * We have consumed all precharges we got in can_attach().
4841                  * We try charge one by one, but don't do any additional
4842                  * charges to mc.to if we have failed in charge once in attach()
4843                  * phase.
4844                  */
4845                 ret = mem_cgroup_do_precharge(1);
4846                 if (!ret)
4847                         goto retry;
4848         }
4849
4850         return ret;
4851 }
4852
4853 static void mem_cgroup_move_charge(struct mm_struct *mm)
4854 {
4855         struct vm_area_struct *vma;
4856
4857         lru_add_drain_all();
4858         /* We've already held the mmap_sem */
4859         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4860                 int ret;
4861                 struct mm_walk mem_cgroup_move_charge_walk = {
4862                         .pmd_entry = mem_cgroup_move_charge_pte_range,
4863                         .mm = mm,
4864                         .private = vma,
4865                 };
4866                 if (is_vm_hugetlb_page(vma))
4867                         continue;
4868                 ret = walk_page_range(vma->vm_start, vma->vm_end,
4869                                                 &mem_cgroup_move_charge_walk);
4870                 if (ret)
4871                         /*
4872                          * means we have consumed all precharges and failed in
4873                          * doing additional charge. Just abandon here.
4874                          */
4875                         break;
4876         }
4877 }
4878
4879 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4880                                 struct cgroup *cont,
4881                                 struct cgroup *old_cont,
4882                                 struct task_struct *p,
4883                                 bool threadgroup)
4884 {
4885         if (!mc.mm)
4886                 /* no need to move charge */
4887                 return;
4888
4889         mem_cgroup_move_charge(mc.mm);
4890         mem_cgroup_clear_mc();
4891 }
4892 #else   /* !CONFIG_MMU */
4893 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4894                                 struct cgroup *cgroup,
4895                                 struct task_struct *p,
4896                                 bool threadgroup)
4897 {
4898         return 0;
4899 }
4900 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4901                                 struct cgroup *cgroup,
4902                                 struct task_struct *p,
4903                                 bool threadgroup)
4904 {
4905 }
4906 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4907                                 struct cgroup *cont,
4908                                 struct cgroup *old_cont,
4909                                 struct task_struct *p,
4910                                 bool threadgroup)
4911 {
4912 }
4913 #endif
4914
4915 struct cgroup_subsys mem_cgroup_subsys = {
4916         .name = "memory",
4917         .subsys_id = mem_cgroup_subsys_id,
4918         .create = mem_cgroup_create,
4919         .pre_destroy = mem_cgroup_pre_destroy,
4920         .destroy = mem_cgroup_destroy,
4921         .populate = mem_cgroup_populate,
4922         .can_attach = mem_cgroup_can_attach,
4923         .cancel_attach = mem_cgroup_cancel_attach,
4924         .attach = mem_cgroup_move_task,
4925         .early_init = 0,
4926         .use_id = 1,
4927 };
4928
4929 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4930 static int __init enable_swap_account(char *s)
4931 {
4932         /* consider enabled if no parameter or 1 is given */
4933         if (!s || !strcmp(s, "1"))
4934                 really_do_swap_account = 1;
4935         else if (!strcmp(s, "0"))
4936                 really_do_swap_account = 0;
4937         return 1;
4938 }
4939 __setup("swapaccount", enable_swap_account);
4940
4941 static int __init disable_swap_account(char *s)
4942 {
4943         enable_swap_account("0");
4944         return 1;
4945 }
4946 __setup("noswapaccount", disable_swap_account);
4947 #endif