memcg: clean up try_charge main loop
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * Memory thresholds
10  * Copyright (C) 2009 Nokia Corporation
11  * Author: Kirill A. Shutemov
12  *
13  * This program is free software; you can redistribute it and/or modify
14  * it under the terms of the GNU General Public License as published by
15  * the Free Software Foundation; either version 2 of the License, or
16  * (at your option) any later version.
17  *
18  * This program is distributed in the hope that it will be useful,
19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
21  * GNU General Public License for more details.
22  */
23
24 #include <linux/res_counter.h>
25 #include <linux/memcontrol.h>
26 #include <linux/cgroup.h>
27 #include <linux/mm.h>
28 #include <linux/hugetlb.h>
29 #include <linux/pagemap.h>
30 #include <linux/smp.h>
31 #include <linux/page-flags.h>
32 #include <linux/backing-dev.h>
33 #include <linux/bit_spinlock.h>
34 #include <linux/rcupdate.h>
35 #include <linux/limits.h>
36 #include <linux/mutex.h>
37 #include <linux/rbtree.h>
38 #include <linux/slab.h>
39 #include <linux/swap.h>
40 #include <linux/swapops.h>
41 #include <linux/spinlock.h>
42 #include <linux/eventfd.h>
43 #include <linux/sort.h>
44 #include <linux/fs.h>
45 #include <linux/seq_file.h>
46 #include <linux/vmalloc.h>
47 #include <linux/mm_inline.h>
48 #include <linux/page_cgroup.h>
49 #include <linux/cpu.h>
50 #include "internal.h"
51
52 #include <asm/uaccess.h>
53
54 #include <trace/events/vmscan.h>
55
56 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
57 #define MEM_CGROUP_RECLAIM_RETRIES      5
58 struct mem_cgroup *root_mem_cgroup __read_mostly;
59
60 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
61 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 1 */
62 int do_swap_account __read_mostly;
63 static int really_do_swap_account __initdata = 1; /* for remember boot option*/
64 #else
65 #define do_swap_account         (0)
66 #endif
67
68 /*
69  * Per memcg event counter is incremented at every pagein/pageout. This counter
70  * is used for trigger some periodic events. This is straightforward and better
71  * than using jiffies etc. to handle periodic memcg event.
72  *
73  * These values will be used as !((event) & ((1 <<(thresh)) - 1))
74  */
75 #define THRESHOLDS_EVENTS_THRESH (7) /* once in 128 */
76 #define SOFTLIMIT_EVENTS_THRESH (10) /* once in 1024 */
77
78 /*
79  * Statistics for memory cgroup.
80  */
81 enum mem_cgroup_stat_index {
82         /*
83          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
84          */
85         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
86         MEM_CGROUP_STAT_RSS,       /* # of pages charged as anon rss */
87         MEM_CGROUP_STAT_FILE_MAPPED,  /* # of pages charged as file rss */
88         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
89         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
90         MEM_CGROUP_STAT_SWAPOUT, /* # of pages, swapped out */
91         MEM_CGROUP_EVENTS,      /* incremented at every  pagein/pageout */
92
93         MEM_CGROUP_STAT_NSTATS,
94 };
95
96 struct mem_cgroup_stat_cpu {
97         s64 count[MEM_CGROUP_STAT_NSTATS];
98 };
99
100 /*
101  * per-zone information in memory controller.
102  */
103 struct mem_cgroup_per_zone {
104         /*
105          * spin_lock to protect the per cgroup LRU
106          */
107         struct list_head        lists[NR_LRU_LISTS];
108         unsigned long           count[NR_LRU_LISTS];
109
110         struct zone_reclaim_stat reclaim_stat;
111         struct rb_node          tree_node;      /* RB tree node */
112         unsigned long long      usage_in_excess;/* Set to the value by which */
113                                                 /* the soft limit is exceeded*/
114         bool                    on_tree;
115         struct mem_cgroup       *mem;           /* Back pointer, we cannot */
116                                                 /* use container_of        */
117 };
118 /* Macro for accessing counter */
119 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
120
121 struct mem_cgroup_per_node {
122         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
123 };
124
125 struct mem_cgroup_lru_info {
126         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
127 };
128
129 /*
130  * Cgroups above their limits are maintained in a RB-Tree, independent of
131  * their hierarchy representation
132  */
133
134 struct mem_cgroup_tree_per_zone {
135         struct rb_root rb_root;
136         spinlock_t lock;
137 };
138
139 struct mem_cgroup_tree_per_node {
140         struct mem_cgroup_tree_per_zone rb_tree_per_zone[MAX_NR_ZONES];
141 };
142
143 struct mem_cgroup_tree {
144         struct mem_cgroup_tree_per_node *rb_tree_per_node[MAX_NUMNODES];
145 };
146
147 static struct mem_cgroup_tree soft_limit_tree __read_mostly;
148
149 struct mem_cgroup_threshold {
150         struct eventfd_ctx *eventfd;
151         u64 threshold;
152 };
153
154 /* For threshold */
155 struct mem_cgroup_threshold_ary {
156         /* An array index points to threshold just below usage. */
157         int current_threshold;
158         /* Size of entries[] */
159         unsigned int size;
160         /* Array of thresholds */
161         struct mem_cgroup_threshold entries[0];
162 };
163
164 struct mem_cgroup_thresholds {
165         /* Primary thresholds array */
166         struct mem_cgroup_threshold_ary *primary;
167         /*
168          * Spare threshold array.
169          * This is needed to make mem_cgroup_unregister_event() "never fail".
170          * It must be able to store at least primary->size - 1 entries.
171          */
172         struct mem_cgroup_threshold_ary *spare;
173 };
174
175 /* for OOM */
176 struct mem_cgroup_eventfd_list {
177         struct list_head list;
178         struct eventfd_ctx *eventfd;
179 };
180
181 static void mem_cgroup_threshold(struct mem_cgroup *mem);
182 static void mem_cgroup_oom_notify(struct mem_cgroup *mem);
183
184 /*
185  * The memory controller data structure. The memory controller controls both
186  * page cache and RSS per cgroup. We would eventually like to provide
187  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
188  * to help the administrator determine what knobs to tune.
189  *
190  * TODO: Add a water mark for the memory controller. Reclaim will begin when
191  * we hit the water mark. May be even add a low water mark, such that
192  * no reclaim occurs from a cgroup at it's low water mark, this is
193  * a feature that will be implemented much later in the future.
194  */
195 struct mem_cgroup {
196         struct cgroup_subsys_state css;
197         /*
198          * the counter to account for memory usage
199          */
200         struct res_counter res;
201         /*
202          * the counter to account for mem+swap usage.
203          */
204         struct res_counter memsw;
205         /*
206          * Per cgroup active and inactive list, similar to the
207          * per zone LRU lists.
208          */
209         struct mem_cgroup_lru_info info;
210
211         /*
212           protect against reclaim related member.
213         */
214         spinlock_t reclaim_param_lock;
215
216         /*
217          * While reclaiming in a hierarchy, we cache the last child we
218          * reclaimed from.
219          */
220         int last_scanned_child;
221         /*
222          * Should the accounting and control be hierarchical, per subtree?
223          */
224         bool use_hierarchy;
225         atomic_t        oom_lock;
226         atomic_t        refcnt;
227
228         unsigned int    swappiness;
229         /* OOM-Killer disable */
230         int             oom_kill_disable;
231
232         /* set when res.limit == memsw.limit */
233         bool            memsw_is_minimum;
234
235         /* protect arrays of thresholds */
236         struct mutex thresholds_lock;
237
238         /* thresholds for memory usage. RCU-protected */
239         struct mem_cgroup_thresholds thresholds;
240
241         /* thresholds for mem+swap usage. RCU-protected */
242         struct mem_cgroup_thresholds memsw_thresholds;
243
244         /* For oom notifier event fd */
245         struct list_head oom_notify;
246
247         /*
248          * Should we move charges of a task when a task is moved into this
249          * mem_cgroup ? And what type of charges should we move ?
250          */
251         unsigned long   move_charge_at_immigrate;
252         /*
253          * percpu counter.
254          */
255         struct mem_cgroup_stat_cpu *stat;
256 };
257
258 /* Stuffs for move charges at task migration. */
259 /*
260  * Types of charges to be moved. "move_charge_at_immitgrate" is treated as a
261  * left-shifted bitmap of these types.
262  */
263 enum move_type {
264         MOVE_CHARGE_TYPE_ANON,  /* private anonymous page and swap of it */
265         MOVE_CHARGE_TYPE_FILE,  /* file page(including tmpfs) and swap of it */
266         NR_MOVE_TYPE,
267 };
268
269 /* "mc" and its members are protected by cgroup_mutex */
270 static struct move_charge_struct {
271         struct mem_cgroup *from;
272         struct mem_cgroup *to;
273         unsigned long precharge;
274         unsigned long moved_charge;
275         unsigned long moved_swap;
276         struct task_struct *moving_task;        /* a task moving charges */
277         wait_queue_head_t waitq;                /* a waitq for other context */
278 } mc = {
279         .waitq = __WAIT_QUEUE_HEAD_INITIALIZER(mc.waitq),
280 };
281
282 static bool move_anon(void)
283 {
284         return test_bit(MOVE_CHARGE_TYPE_ANON,
285                                         &mc.to->move_charge_at_immigrate);
286 }
287
288 static bool move_file(void)
289 {
290         return test_bit(MOVE_CHARGE_TYPE_FILE,
291                                         &mc.to->move_charge_at_immigrate);
292 }
293
294 /*
295  * Maximum loops in mem_cgroup_hierarchical_reclaim(), used for soft
296  * limit reclaim to prevent infinite loops, if they ever occur.
297  */
298 #define MEM_CGROUP_MAX_RECLAIM_LOOPS            (100)
299 #define MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS (2)
300
301 enum charge_type {
302         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
303         MEM_CGROUP_CHARGE_TYPE_MAPPED,
304         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
305         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
306         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
307         MEM_CGROUP_CHARGE_TYPE_DROP,    /* a page was unused swap cache */
308         NR_CHARGE_TYPE,
309 };
310
311 /* only for here (for easy reading.) */
312 #define PCGF_CACHE      (1UL << PCG_CACHE)
313 #define PCGF_USED       (1UL << PCG_USED)
314 #define PCGF_LOCK       (1UL << PCG_LOCK)
315 /* Not used, but added here for completeness */
316 #define PCGF_ACCT       (1UL << PCG_ACCT)
317
318 /* for encoding cft->private value on file */
319 #define _MEM                    (0)
320 #define _MEMSWAP                (1)
321 #define _OOM_TYPE               (2)
322 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
323 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
324 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
325 /* Used for OOM nofiier */
326 #define OOM_CONTROL             (0)
327
328 /*
329  * Reclaim flags for mem_cgroup_hierarchical_reclaim
330  */
331 #define MEM_CGROUP_RECLAIM_NOSWAP_BIT   0x0
332 #define MEM_CGROUP_RECLAIM_NOSWAP       (1 << MEM_CGROUP_RECLAIM_NOSWAP_BIT)
333 #define MEM_CGROUP_RECLAIM_SHRINK_BIT   0x1
334 #define MEM_CGROUP_RECLAIM_SHRINK       (1 << MEM_CGROUP_RECLAIM_SHRINK_BIT)
335 #define MEM_CGROUP_RECLAIM_SOFT_BIT     0x2
336 #define MEM_CGROUP_RECLAIM_SOFT         (1 << MEM_CGROUP_RECLAIM_SOFT_BIT)
337
338 static void mem_cgroup_get(struct mem_cgroup *mem);
339 static void mem_cgroup_put(struct mem_cgroup *mem);
340 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
341 static void drain_all_stock_async(void);
342
343 static struct mem_cgroup_per_zone *
344 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
345 {
346         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
347 }
348
349 struct cgroup_subsys_state *mem_cgroup_css(struct mem_cgroup *mem)
350 {
351         return &mem->css;
352 }
353
354 static struct mem_cgroup_per_zone *
355 page_cgroup_zoneinfo(struct page_cgroup *pc)
356 {
357         struct mem_cgroup *mem = pc->mem_cgroup;
358         int nid = page_cgroup_nid(pc);
359         int zid = page_cgroup_zid(pc);
360
361         if (!mem)
362                 return NULL;
363
364         return mem_cgroup_zoneinfo(mem, nid, zid);
365 }
366
367 static struct mem_cgroup_tree_per_zone *
368 soft_limit_tree_node_zone(int nid, int zid)
369 {
370         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
371 }
372
373 static struct mem_cgroup_tree_per_zone *
374 soft_limit_tree_from_page(struct page *page)
375 {
376         int nid = page_to_nid(page);
377         int zid = page_zonenum(page);
378
379         return &soft_limit_tree.rb_tree_per_node[nid]->rb_tree_per_zone[zid];
380 }
381
382 static void
383 __mem_cgroup_insert_exceeded(struct mem_cgroup *mem,
384                                 struct mem_cgroup_per_zone *mz,
385                                 struct mem_cgroup_tree_per_zone *mctz,
386                                 unsigned long long new_usage_in_excess)
387 {
388         struct rb_node **p = &mctz->rb_root.rb_node;
389         struct rb_node *parent = NULL;
390         struct mem_cgroup_per_zone *mz_node;
391
392         if (mz->on_tree)
393                 return;
394
395         mz->usage_in_excess = new_usage_in_excess;
396         if (!mz->usage_in_excess)
397                 return;
398         while (*p) {
399                 parent = *p;
400                 mz_node = rb_entry(parent, struct mem_cgroup_per_zone,
401                                         tree_node);
402                 if (mz->usage_in_excess < mz_node->usage_in_excess)
403                         p = &(*p)->rb_left;
404                 /*
405                  * We can't avoid mem cgroups that are over their soft
406                  * limit by the same amount
407                  */
408                 else if (mz->usage_in_excess >= mz_node->usage_in_excess)
409                         p = &(*p)->rb_right;
410         }
411         rb_link_node(&mz->tree_node, parent, p);
412         rb_insert_color(&mz->tree_node, &mctz->rb_root);
413         mz->on_tree = true;
414 }
415
416 static void
417 __mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
418                                 struct mem_cgroup_per_zone *mz,
419                                 struct mem_cgroup_tree_per_zone *mctz)
420 {
421         if (!mz->on_tree)
422                 return;
423         rb_erase(&mz->tree_node, &mctz->rb_root);
424         mz->on_tree = false;
425 }
426
427 static void
428 mem_cgroup_remove_exceeded(struct mem_cgroup *mem,
429                                 struct mem_cgroup_per_zone *mz,
430                                 struct mem_cgroup_tree_per_zone *mctz)
431 {
432         spin_lock(&mctz->lock);
433         __mem_cgroup_remove_exceeded(mem, mz, mctz);
434         spin_unlock(&mctz->lock);
435 }
436
437
438 static void mem_cgroup_update_tree(struct mem_cgroup *mem, struct page *page)
439 {
440         unsigned long long excess;
441         struct mem_cgroup_per_zone *mz;
442         struct mem_cgroup_tree_per_zone *mctz;
443         int nid = page_to_nid(page);
444         int zid = page_zonenum(page);
445         mctz = soft_limit_tree_from_page(page);
446
447         /*
448          * Necessary to update all ancestors when hierarchy is used.
449          * because their event counter is not touched.
450          */
451         for (; mem; mem = parent_mem_cgroup(mem)) {
452                 mz = mem_cgroup_zoneinfo(mem, nid, zid);
453                 excess = res_counter_soft_limit_excess(&mem->res);
454                 /*
455                  * We have to update the tree if mz is on RB-tree or
456                  * mem is over its softlimit.
457                  */
458                 if (excess || mz->on_tree) {
459                         spin_lock(&mctz->lock);
460                         /* if on-tree, remove it */
461                         if (mz->on_tree)
462                                 __mem_cgroup_remove_exceeded(mem, mz, mctz);
463                         /*
464                          * Insert again. mz->usage_in_excess will be updated.
465                          * If excess is 0, no tree ops.
466                          */
467                         __mem_cgroup_insert_exceeded(mem, mz, mctz, excess);
468                         spin_unlock(&mctz->lock);
469                 }
470         }
471 }
472
473 static void mem_cgroup_remove_from_trees(struct mem_cgroup *mem)
474 {
475         int node, zone;
476         struct mem_cgroup_per_zone *mz;
477         struct mem_cgroup_tree_per_zone *mctz;
478
479         for_each_node_state(node, N_POSSIBLE) {
480                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
481                         mz = mem_cgroup_zoneinfo(mem, node, zone);
482                         mctz = soft_limit_tree_node_zone(node, zone);
483                         mem_cgroup_remove_exceeded(mem, mz, mctz);
484                 }
485         }
486 }
487
488 static inline unsigned long mem_cgroup_get_excess(struct mem_cgroup *mem)
489 {
490         return res_counter_soft_limit_excess(&mem->res) >> PAGE_SHIFT;
491 }
492
493 static struct mem_cgroup_per_zone *
494 __mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
495 {
496         struct rb_node *rightmost = NULL;
497         struct mem_cgroup_per_zone *mz;
498
499 retry:
500         mz = NULL;
501         rightmost = rb_last(&mctz->rb_root);
502         if (!rightmost)
503                 goto done;              /* Nothing to reclaim from */
504
505         mz = rb_entry(rightmost, struct mem_cgroup_per_zone, tree_node);
506         /*
507          * Remove the node now but someone else can add it back,
508          * we will to add it back at the end of reclaim to its correct
509          * position in the tree.
510          */
511         __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
512         if (!res_counter_soft_limit_excess(&mz->mem->res) ||
513                 !css_tryget(&mz->mem->css))
514                 goto retry;
515 done:
516         return mz;
517 }
518
519 static struct mem_cgroup_per_zone *
520 mem_cgroup_largest_soft_limit_node(struct mem_cgroup_tree_per_zone *mctz)
521 {
522         struct mem_cgroup_per_zone *mz;
523
524         spin_lock(&mctz->lock);
525         mz = __mem_cgroup_largest_soft_limit_node(mctz);
526         spin_unlock(&mctz->lock);
527         return mz;
528 }
529
530 static s64 mem_cgroup_read_stat(struct mem_cgroup *mem,
531                 enum mem_cgroup_stat_index idx)
532 {
533         int cpu;
534         s64 val = 0;
535
536         for_each_possible_cpu(cpu)
537                 val += per_cpu(mem->stat->count[idx], cpu);
538         return val;
539 }
540
541 static s64 mem_cgroup_local_usage(struct mem_cgroup *mem)
542 {
543         s64 ret;
544
545         ret = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
546         ret += mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
547         return ret;
548 }
549
550 static void mem_cgroup_swap_statistics(struct mem_cgroup *mem,
551                                          bool charge)
552 {
553         int val = (charge) ? 1 : -1;
554         this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_SWAPOUT], val);
555 }
556
557 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
558                                          struct page_cgroup *pc,
559                                          bool charge)
560 {
561         int val = (charge) ? 1 : -1;
562
563         preempt_disable();
564
565         if (PageCgroupCache(pc))
566                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_CACHE], val);
567         else
568                 __this_cpu_add(mem->stat->count[MEM_CGROUP_STAT_RSS], val);
569
570         if (charge)
571                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGIN_COUNT]);
572         else
573                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_PGPGOUT_COUNT]);
574         __this_cpu_inc(mem->stat->count[MEM_CGROUP_EVENTS]);
575
576         preempt_enable();
577 }
578
579 static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
580                                         enum lru_list idx)
581 {
582         int nid, zid;
583         struct mem_cgroup_per_zone *mz;
584         u64 total = 0;
585
586         for_each_online_node(nid)
587                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
588                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
589                         total += MEM_CGROUP_ZSTAT(mz, idx);
590                 }
591         return total;
592 }
593
594 static bool __memcg_event_check(struct mem_cgroup *mem, int event_mask_shift)
595 {
596         s64 val;
597
598         val = this_cpu_read(mem->stat->count[MEM_CGROUP_EVENTS]);
599
600         return !(val & ((1 << event_mask_shift) - 1));
601 }
602
603 /*
604  * Check events in order.
605  *
606  */
607 static void memcg_check_events(struct mem_cgroup *mem, struct page *page)
608 {
609         /* threshold event is triggered in finer grain than soft limit */
610         if (unlikely(__memcg_event_check(mem, THRESHOLDS_EVENTS_THRESH))) {
611                 mem_cgroup_threshold(mem);
612                 if (unlikely(__memcg_event_check(mem, SOFTLIMIT_EVENTS_THRESH)))
613                         mem_cgroup_update_tree(mem, page);
614         }
615 }
616
617 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
618 {
619         return container_of(cgroup_subsys_state(cont,
620                                 mem_cgroup_subsys_id), struct mem_cgroup,
621                                 css);
622 }
623
624 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
625 {
626         /*
627          * mm_update_next_owner() may clear mm->owner to NULL
628          * if it races with swapoff, page migration, etc.
629          * So this can be called with p == NULL.
630          */
631         if (unlikely(!p))
632                 return NULL;
633
634         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
635                                 struct mem_cgroup, css);
636 }
637
638 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
639 {
640         struct mem_cgroup *mem = NULL;
641
642         if (!mm)
643                 return NULL;
644         /*
645          * Because we have no locks, mm->owner's may be being moved to other
646          * cgroup. We use css_tryget() here even if this looks
647          * pessimistic (rather than adding locks here).
648          */
649         rcu_read_lock();
650         do {
651                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
652                 if (unlikely(!mem))
653                         break;
654         } while (!css_tryget(&mem->css));
655         rcu_read_unlock();
656         return mem;
657 }
658
659 /*
660  * Call callback function against all cgroup under hierarchy tree.
661  */
662 static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
663                           int (*func)(struct mem_cgroup *, void *))
664 {
665         int found, ret, nextid;
666         struct cgroup_subsys_state *css;
667         struct mem_cgroup *mem;
668
669         if (!root->use_hierarchy)
670                 return (*func)(root, data);
671
672         nextid = 1;
673         do {
674                 ret = 0;
675                 mem = NULL;
676
677                 rcu_read_lock();
678                 css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
679                                    &found);
680                 if (css && css_tryget(css))
681                         mem = container_of(css, struct mem_cgroup, css);
682                 rcu_read_unlock();
683
684                 if (mem) {
685                         ret = (*func)(mem, data);
686                         css_put(&mem->css);
687                 }
688                 nextid = found + 1;
689         } while (!ret && css);
690
691         return ret;
692 }
693
694 static inline bool mem_cgroup_is_root(struct mem_cgroup *mem)
695 {
696         return (mem == root_mem_cgroup);
697 }
698
699 /*
700  * Following LRU functions are allowed to be used without PCG_LOCK.
701  * Operations are called by routine of global LRU independently from memcg.
702  * What we have to take care of here is validness of pc->mem_cgroup.
703  *
704  * Changes to pc->mem_cgroup happens when
705  * 1. charge
706  * 2. moving account
707  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
708  * It is added to LRU before charge.
709  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
710  * When moving account, the page is not on LRU. It's isolated.
711  */
712
713 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
714 {
715         struct page_cgroup *pc;
716         struct mem_cgroup_per_zone *mz;
717
718         if (mem_cgroup_disabled())
719                 return;
720         pc = lookup_page_cgroup(page);
721         /* can happen while we handle swapcache. */
722         if (!TestClearPageCgroupAcctLRU(pc))
723                 return;
724         VM_BUG_ON(!pc->mem_cgroup);
725         /*
726          * We don't check PCG_USED bit. It's cleared when the "page" is finally
727          * removed from global LRU.
728          */
729         mz = page_cgroup_zoneinfo(pc);
730         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
731         if (mem_cgroup_is_root(pc->mem_cgroup))
732                 return;
733         VM_BUG_ON(list_empty(&pc->lru));
734         list_del_init(&pc->lru);
735         return;
736 }
737
738 void mem_cgroup_del_lru(struct page *page)
739 {
740         mem_cgroup_del_lru_list(page, page_lru(page));
741 }
742
743 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
744 {
745         struct mem_cgroup_per_zone *mz;
746         struct page_cgroup *pc;
747
748         if (mem_cgroup_disabled())
749                 return;
750
751         pc = lookup_page_cgroup(page);
752         /*
753          * Used bit is set without atomic ops but after smp_wmb().
754          * For making pc->mem_cgroup visible, insert smp_rmb() here.
755          */
756         smp_rmb();
757         /* unused or root page is not rotated. */
758         if (!PageCgroupUsed(pc) || mem_cgroup_is_root(pc->mem_cgroup))
759                 return;
760         mz = page_cgroup_zoneinfo(pc);
761         list_move(&pc->lru, &mz->lists[lru]);
762 }
763
764 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
765 {
766         struct page_cgroup *pc;
767         struct mem_cgroup_per_zone *mz;
768
769         if (mem_cgroup_disabled())
770                 return;
771         pc = lookup_page_cgroup(page);
772         VM_BUG_ON(PageCgroupAcctLRU(pc));
773         /*
774          * Used bit is set without atomic ops but after smp_wmb().
775          * For making pc->mem_cgroup visible, insert smp_rmb() here.
776          */
777         smp_rmb();
778         if (!PageCgroupUsed(pc))
779                 return;
780
781         mz = page_cgroup_zoneinfo(pc);
782         MEM_CGROUP_ZSTAT(mz, lru) += 1;
783         SetPageCgroupAcctLRU(pc);
784         if (mem_cgroup_is_root(pc->mem_cgroup))
785                 return;
786         list_add(&pc->lru, &mz->lists[lru]);
787 }
788
789 /*
790  * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
791  * lru because the page may.be reused after it's fully uncharged (because of
792  * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
793  * it again. This function is only used to charge SwapCache. It's done under
794  * lock_page and expected that zone->lru_lock is never held.
795  */
796 static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
797 {
798         unsigned long flags;
799         struct zone *zone = page_zone(page);
800         struct page_cgroup *pc = lookup_page_cgroup(page);
801
802         spin_lock_irqsave(&zone->lru_lock, flags);
803         /*
804          * Forget old LRU when this page_cgroup is *not* used. This Used bit
805          * is guarded by lock_page() because the page is SwapCache.
806          */
807         if (!PageCgroupUsed(pc))
808                 mem_cgroup_del_lru_list(page, page_lru(page));
809         spin_unlock_irqrestore(&zone->lru_lock, flags);
810 }
811
812 static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
813 {
814         unsigned long flags;
815         struct zone *zone = page_zone(page);
816         struct page_cgroup *pc = lookup_page_cgroup(page);
817
818         spin_lock_irqsave(&zone->lru_lock, flags);
819         /* link when the page is linked to LRU but page_cgroup isn't */
820         if (PageLRU(page) && !PageCgroupAcctLRU(pc))
821                 mem_cgroup_add_lru_list(page, page_lru(page));
822         spin_unlock_irqrestore(&zone->lru_lock, flags);
823 }
824
825
826 void mem_cgroup_move_lists(struct page *page,
827                            enum lru_list from, enum lru_list to)
828 {
829         if (mem_cgroup_disabled())
830                 return;
831         mem_cgroup_del_lru_list(page, from);
832         mem_cgroup_add_lru_list(page, to);
833 }
834
835 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
836 {
837         int ret;
838         struct mem_cgroup *curr = NULL;
839
840         task_lock(task);
841         rcu_read_lock();
842         curr = try_get_mem_cgroup_from_mm(task->mm);
843         rcu_read_unlock();
844         task_unlock(task);
845         if (!curr)
846                 return 0;
847         /*
848          * We should check use_hierarchy of "mem" not "curr". Because checking
849          * use_hierarchy of "curr" here make this function true if hierarchy is
850          * enabled in "curr" and "curr" is a child of "mem" in *cgroup*
851          * hierarchy(even if use_hierarchy is disabled in "mem").
852          */
853         if (mem->use_hierarchy)
854                 ret = css_is_ancestor(&curr->css, &mem->css);
855         else
856                 ret = (curr == mem);
857         css_put(&curr->css);
858         return ret;
859 }
860
861 static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
862 {
863         unsigned long active;
864         unsigned long inactive;
865         unsigned long gb;
866         unsigned long inactive_ratio;
867
868         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
869         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
870
871         gb = (inactive + active) >> (30 - PAGE_SHIFT);
872         if (gb)
873                 inactive_ratio = int_sqrt(10 * gb);
874         else
875                 inactive_ratio = 1;
876
877         if (present_pages) {
878                 present_pages[0] = inactive;
879                 present_pages[1] = active;
880         }
881
882         return inactive_ratio;
883 }
884
885 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
886 {
887         unsigned long active;
888         unsigned long inactive;
889         unsigned long present_pages[2];
890         unsigned long inactive_ratio;
891
892         inactive_ratio = calc_inactive_ratio(memcg, present_pages);
893
894         inactive = present_pages[0];
895         active = present_pages[1];
896
897         if (inactive * inactive_ratio < active)
898                 return 1;
899
900         return 0;
901 }
902
903 int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
904 {
905         unsigned long active;
906         unsigned long inactive;
907
908         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
909         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
910
911         return (active > inactive);
912 }
913
914 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
915                                        struct zone *zone,
916                                        enum lru_list lru)
917 {
918         int nid = zone->zone_pgdat->node_id;
919         int zid = zone_idx(zone);
920         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
921
922         return MEM_CGROUP_ZSTAT(mz, lru);
923 }
924
925 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
926                                                       struct zone *zone)
927 {
928         int nid = zone->zone_pgdat->node_id;
929         int zid = zone_idx(zone);
930         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
931
932         return &mz->reclaim_stat;
933 }
934
935 struct zone_reclaim_stat *
936 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
937 {
938         struct page_cgroup *pc;
939         struct mem_cgroup_per_zone *mz;
940
941         if (mem_cgroup_disabled())
942                 return NULL;
943
944         pc = lookup_page_cgroup(page);
945         /*
946          * Used bit is set without atomic ops but after smp_wmb().
947          * For making pc->mem_cgroup visible, insert smp_rmb() here.
948          */
949         smp_rmb();
950         if (!PageCgroupUsed(pc))
951                 return NULL;
952
953         mz = page_cgroup_zoneinfo(pc);
954         if (!mz)
955                 return NULL;
956
957         return &mz->reclaim_stat;
958 }
959
960 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
961                                         struct list_head *dst,
962                                         unsigned long *scanned, int order,
963                                         int mode, struct zone *z,
964                                         struct mem_cgroup *mem_cont,
965                                         int active, int file)
966 {
967         unsigned long nr_taken = 0;
968         struct page *page;
969         unsigned long scan;
970         LIST_HEAD(pc_list);
971         struct list_head *src;
972         struct page_cgroup *pc, *tmp;
973         int nid = z->zone_pgdat->node_id;
974         int zid = zone_idx(z);
975         struct mem_cgroup_per_zone *mz;
976         int lru = LRU_FILE * file + active;
977         int ret;
978
979         BUG_ON(!mem_cont);
980         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
981         src = &mz->lists[lru];
982
983         scan = 0;
984         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
985                 if (scan >= nr_to_scan)
986                         break;
987
988                 page = pc->page;
989                 if (unlikely(!PageCgroupUsed(pc)))
990                         continue;
991                 if (unlikely(!PageLRU(page)))
992                         continue;
993
994                 scan++;
995                 ret = __isolate_lru_page(page, mode, file);
996                 switch (ret) {
997                 case 0:
998                         list_move(&page->lru, dst);
999                         mem_cgroup_del_lru(page);
1000                         nr_taken++;
1001                         break;
1002                 case -EBUSY:
1003                         /* we don't affect global LRU but rotate in our LRU */
1004                         mem_cgroup_rotate_lru_list(page, page_lru(page));
1005                         break;
1006                 default:
1007                         break;
1008                 }
1009         }
1010
1011         *scanned = scan;
1012
1013         trace_mm_vmscan_memcg_isolate(0, nr_to_scan, scan, nr_taken,
1014                                       0, 0, 0, mode);
1015
1016         return nr_taken;
1017 }
1018
1019 #define mem_cgroup_from_res_counter(counter, member)    \
1020         container_of(counter, struct mem_cgroup, member)
1021
1022 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
1023 {
1024         if (do_swap_account) {
1025                 if (res_counter_check_under_limit(&mem->res) &&
1026                         res_counter_check_under_limit(&mem->memsw))
1027                         return true;
1028         } else
1029                 if (res_counter_check_under_limit(&mem->res))
1030                         return true;
1031         return false;
1032 }
1033
1034 static unsigned int get_swappiness(struct mem_cgroup *memcg)
1035 {
1036         struct cgroup *cgrp = memcg->css.cgroup;
1037         unsigned int swappiness;
1038
1039         /* root ? */
1040         if (cgrp->parent == NULL)
1041                 return vm_swappiness;
1042
1043         spin_lock(&memcg->reclaim_param_lock);
1044         swappiness = memcg->swappiness;
1045         spin_unlock(&memcg->reclaim_param_lock);
1046
1047         return swappiness;
1048 }
1049
1050 /* A routine for testing mem is not under move_account */
1051
1052 static bool mem_cgroup_under_move(struct mem_cgroup *mem)
1053 {
1054         struct mem_cgroup *from = mc.from;
1055         struct mem_cgroup *to = mc.to;
1056         bool ret = false;
1057
1058         if (from == mem || to == mem)
1059                 return true;
1060
1061         if (!from || !to || !mem->use_hierarchy)
1062                 return false;
1063
1064         rcu_read_lock();
1065         if (css_tryget(&from->css)) {
1066                 ret = css_is_ancestor(&from->css, &mem->css);
1067                 css_put(&from->css);
1068         }
1069         if (!ret && css_tryget(&to->css)) {
1070                 ret = css_is_ancestor(&to->css, &mem->css);
1071                 css_put(&to->css);
1072         }
1073         rcu_read_unlock();
1074         return ret;
1075 }
1076
1077 static bool mem_cgroup_wait_acct_move(struct mem_cgroup *mem)
1078 {
1079         if (mc.moving_task && current != mc.moving_task) {
1080                 if (mem_cgroup_under_move(mem)) {
1081                         DEFINE_WAIT(wait);
1082                         prepare_to_wait(&mc.waitq, &wait, TASK_INTERRUPTIBLE);
1083                         /* moving charge context might have finished. */
1084                         if (mc.moving_task)
1085                                 schedule();
1086                         finish_wait(&mc.waitq, &wait);
1087                         return true;
1088                 }
1089         }
1090         return false;
1091 }
1092
1093 static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
1094 {
1095         int *val = data;
1096         (*val)++;
1097         return 0;
1098 }
1099
1100 /**
1101  * mem_cgroup_print_oom_info: Called from OOM with tasklist_lock held in read mode.
1102  * @memcg: The memory cgroup that went over limit
1103  * @p: Task that is going to be killed
1104  *
1105  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
1106  * enabled
1107  */
1108 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
1109 {
1110         struct cgroup *task_cgrp;
1111         struct cgroup *mem_cgrp;
1112         /*
1113          * Need a buffer in BSS, can't rely on allocations. The code relies
1114          * on the assumption that OOM is serialized for memory controller.
1115          * If this assumption is broken, revisit this code.
1116          */
1117         static char memcg_name[PATH_MAX];
1118         int ret;
1119
1120         if (!memcg || !p)
1121                 return;
1122
1123
1124         rcu_read_lock();
1125
1126         mem_cgrp = memcg->css.cgroup;
1127         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
1128
1129         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
1130         if (ret < 0) {
1131                 /*
1132                  * Unfortunately, we are unable to convert to a useful name
1133                  * But we'll still print out the usage information
1134                  */
1135                 rcu_read_unlock();
1136                 goto done;
1137         }
1138         rcu_read_unlock();
1139
1140         printk(KERN_INFO "Task in %s killed", memcg_name);
1141
1142         rcu_read_lock();
1143         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
1144         if (ret < 0) {
1145                 rcu_read_unlock();
1146                 goto done;
1147         }
1148         rcu_read_unlock();
1149
1150         /*
1151          * Continues from above, so we don't need an KERN_ level
1152          */
1153         printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
1154 done:
1155
1156         printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
1157                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
1158                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
1159                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
1160         printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
1161                 "failcnt %llu\n",
1162                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
1163                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
1164                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
1165 }
1166
1167 /*
1168  * This function returns the number of memcg under hierarchy tree. Returns
1169  * 1(self count) if no children.
1170  */
1171 static int mem_cgroup_count_children(struct mem_cgroup *mem)
1172 {
1173         int num = 0;
1174         mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
1175         return num;
1176 }
1177
1178 /*
1179  * Return the memory (and swap, if configured) limit for a memcg.
1180  */
1181 u64 mem_cgroup_get_limit(struct mem_cgroup *memcg)
1182 {
1183         u64 limit;
1184         u64 memsw;
1185
1186         limit = res_counter_read_u64(&memcg->res, RES_LIMIT) +
1187                         total_swap_pages;
1188         memsw = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1189         /*
1190          * If memsw is finite and limits the amount of swap space available
1191          * to this memcg, return that limit.
1192          */
1193         return min(limit, memsw);
1194 }
1195
1196 /*
1197  * Visit the first child (need not be the first child as per the ordering
1198  * of the cgroup list, since we track last_scanned_child) of @mem and use
1199  * that to reclaim free pages from.
1200  */
1201 static struct mem_cgroup *
1202 mem_cgroup_select_victim(struct mem_cgroup *root_mem)
1203 {
1204         struct mem_cgroup *ret = NULL;
1205         struct cgroup_subsys_state *css;
1206         int nextid, found;
1207
1208         if (!root_mem->use_hierarchy) {
1209                 css_get(&root_mem->css);
1210                 ret = root_mem;
1211         }
1212
1213         while (!ret) {
1214                 rcu_read_lock();
1215                 nextid = root_mem->last_scanned_child + 1;
1216                 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
1217                                    &found);
1218                 if (css && css_tryget(css))
1219                         ret = container_of(css, struct mem_cgroup, css);
1220
1221                 rcu_read_unlock();
1222                 /* Updates scanning parameter */
1223                 spin_lock(&root_mem->reclaim_param_lock);
1224                 if (!css) {
1225                         /* this means start scan from ID:1 */
1226                         root_mem->last_scanned_child = 0;
1227                 } else
1228                         root_mem->last_scanned_child = found;
1229                 spin_unlock(&root_mem->reclaim_param_lock);
1230         }
1231
1232         return ret;
1233 }
1234
1235 /*
1236  * Scan the hierarchy if needed to reclaim memory. We remember the last child
1237  * we reclaimed from, so that we don't end up penalizing one child extensively
1238  * based on its position in the children list.
1239  *
1240  * root_mem is the original ancestor that we've been reclaim from.
1241  *
1242  * We give up and return to the caller when we visit root_mem twice.
1243  * (other groups can be removed while we're walking....)
1244  *
1245  * If shrink==true, for avoiding to free too much, this returns immedieately.
1246  */
1247 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
1248                                                 struct zone *zone,
1249                                                 gfp_t gfp_mask,
1250                                                 unsigned long reclaim_options)
1251 {
1252         struct mem_cgroup *victim;
1253         int ret, total = 0;
1254         int loop = 0;
1255         bool noswap = reclaim_options & MEM_CGROUP_RECLAIM_NOSWAP;
1256         bool shrink = reclaim_options & MEM_CGROUP_RECLAIM_SHRINK;
1257         bool check_soft = reclaim_options & MEM_CGROUP_RECLAIM_SOFT;
1258         unsigned long excess = mem_cgroup_get_excess(root_mem);
1259
1260         /* If memsw_is_minimum==1, swap-out is of-no-use. */
1261         if (root_mem->memsw_is_minimum)
1262                 noswap = true;
1263
1264         while (1) {
1265                 victim = mem_cgroup_select_victim(root_mem);
1266                 if (victim == root_mem) {
1267                         loop++;
1268                         if (loop >= 1)
1269                                 drain_all_stock_async();
1270                         if (loop >= 2) {
1271                                 /*
1272                                  * If we have not been able to reclaim
1273                                  * anything, it might because there are
1274                                  * no reclaimable pages under this hierarchy
1275                                  */
1276                                 if (!check_soft || !total) {
1277                                         css_put(&victim->css);
1278                                         break;
1279                                 }
1280                                 /*
1281                                  * We want to do more targetted reclaim.
1282                                  * excess >> 2 is not to excessive so as to
1283                                  * reclaim too much, nor too less that we keep
1284                                  * coming back to reclaim from this cgroup
1285                                  */
1286                                 if (total >= (excess >> 2) ||
1287                                         (loop > MEM_CGROUP_MAX_RECLAIM_LOOPS)) {
1288                                         css_put(&victim->css);
1289                                         break;
1290                                 }
1291                         }
1292                 }
1293                 if (!mem_cgroup_local_usage(victim)) {
1294                         /* this cgroup's local usage == 0 */
1295                         css_put(&victim->css);
1296                         continue;
1297                 }
1298                 /* we use swappiness of local cgroup */
1299                 if (check_soft)
1300                         ret = mem_cgroup_shrink_node_zone(victim, gfp_mask,
1301                                 noswap, get_swappiness(victim), zone,
1302                                 zone->zone_pgdat->node_id);
1303                 else
1304                         ret = try_to_free_mem_cgroup_pages(victim, gfp_mask,
1305                                                 noswap, get_swappiness(victim));
1306                 css_put(&victim->css);
1307                 /*
1308                  * At shrinking usage, we can't check we should stop here or
1309                  * reclaim more. It's depends on callers. last_scanned_child
1310                  * will work enough for keeping fairness under tree.
1311                  */
1312                 if (shrink)
1313                         return ret;
1314                 total += ret;
1315                 if (check_soft) {
1316                         if (res_counter_check_under_soft_limit(&root_mem->res))
1317                                 return total;
1318                 } else if (mem_cgroup_check_under_limit(root_mem))
1319                         return 1 + total;
1320         }
1321         return total;
1322 }
1323
1324 static int mem_cgroup_oom_lock_cb(struct mem_cgroup *mem, void *data)
1325 {
1326         int *val = (int *)data;
1327         int x;
1328         /*
1329          * Logically, we can stop scanning immediately when we find
1330          * a memcg is already locked. But condidering unlock ops and
1331          * creation/removal of memcg, scan-all is simple operation.
1332          */
1333         x = atomic_inc_return(&mem->oom_lock);
1334         *val = max(x, *val);
1335         return 0;
1336 }
1337 /*
1338  * Check OOM-Killer is already running under our hierarchy.
1339  * If someone is running, return false.
1340  */
1341 static bool mem_cgroup_oom_lock(struct mem_cgroup *mem)
1342 {
1343         int lock_count = 0;
1344
1345         mem_cgroup_walk_tree(mem, &lock_count, mem_cgroup_oom_lock_cb);
1346
1347         if (lock_count == 1)
1348                 return true;
1349         return false;
1350 }
1351
1352 static int mem_cgroup_oom_unlock_cb(struct mem_cgroup *mem, void *data)
1353 {
1354         /*
1355          * When a new child is created while the hierarchy is under oom,
1356          * mem_cgroup_oom_lock() may not be called. We have to use
1357          * atomic_add_unless() here.
1358          */
1359         atomic_add_unless(&mem->oom_lock, -1, 0);
1360         return 0;
1361 }
1362
1363 static void mem_cgroup_oom_unlock(struct mem_cgroup *mem)
1364 {
1365         mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_unlock_cb);
1366 }
1367
1368 static DEFINE_MUTEX(memcg_oom_mutex);
1369 static DECLARE_WAIT_QUEUE_HEAD(memcg_oom_waitq);
1370
1371 struct oom_wait_info {
1372         struct mem_cgroup *mem;
1373         wait_queue_t    wait;
1374 };
1375
1376 static int memcg_oom_wake_function(wait_queue_t *wait,
1377         unsigned mode, int sync, void *arg)
1378 {
1379         struct mem_cgroup *wake_mem = (struct mem_cgroup *)arg;
1380         struct oom_wait_info *oom_wait_info;
1381
1382         oom_wait_info = container_of(wait, struct oom_wait_info, wait);
1383
1384         if (oom_wait_info->mem == wake_mem)
1385                 goto wakeup;
1386         /* if no hierarchy, no match */
1387         if (!oom_wait_info->mem->use_hierarchy || !wake_mem->use_hierarchy)
1388                 return 0;
1389         /*
1390          * Both of oom_wait_info->mem and wake_mem are stable under us.
1391          * Then we can use css_is_ancestor without taking care of RCU.
1392          */
1393         if (!css_is_ancestor(&oom_wait_info->mem->css, &wake_mem->css) &&
1394             !css_is_ancestor(&wake_mem->css, &oom_wait_info->mem->css))
1395                 return 0;
1396
1397 wakeup:
1398         return autoremove_wake_function(wait, mode, sync, arg);
1399 }
1400
1401 static void memcg_wakeup_oom(struct mem_cgroup *mem)
1402 {
1403         /* for filtering, pass "mem" as argument. */
1404         __wake_up(&memcg_oom_waitq, TASK_NORMAL, 0, mem);
1405 }
1406
1407 static void memcg_oom_recover(struct mem_cgroup *mem)
1408 {
1409         if (atomic_read(&mem->oom_lock))
1410                 memcg_wakeup_oom(mem);
1411 }
1412
1413 /*
1414  * try to call OOM killer. returns false if we should exit memory-reclaim loop.
1415  */
1416 bool mem_cgroup_handle_oom(struct mem_cgroup *mem, gfp_t mask)
1417 {
1418         struct oom_wait_info owait;
1419         bool locked, need_to_kill;
1420
1421         owait.mem = mem;
1422         owait.wait.flags = 0;
1423         owait.wait.func = memcg_oom_wake_function;
1424         owait.wait.private = current;
1425         INIT_LIST_HEAD(&owait.wait.task_list);
1426         need_to_kill = true;
1427         /* At first, try to OOM lock hierarchy under mem.*/
1428         mutex_lock(&memcg_oom_mutex);
1429         locked = mem_cgroup_oom_lock(mem);
1430         /*
1431          * Even if signal_pending(), we can't quit charge() loop without
1432          * accounting. So, UNINTERRUPTIBLE is appropriate. But SIGKILL
1433          * under OOM is always welcomed, use TASK_KILLABLE here.
1434          */
1435         prepare_to_wait(&memcg_oom_waitq, &owait.wait, TASK_KILLABLE);
1436         if (!locked || mem->oom_kill_disable)
1437                 need_to_kill = false;
1438         if (locked)
1439                 mem_cgroup_oom_notify(mem);
1440         mutex_unlock(&memcg_oom_mutex);
1441
1442         if (need_to_kill) {
1443                 finish_wait(&memcg_oom_waitq, &owait.wait);
1444                 mem_cgroup_out_of_memory(mem, mask);
1445         } else {
1446                 schedule();
1447                 finish_wait(&memcg_oom_waitq, &owait.wait);
1448         }
1449         mutex_lock(&memcg_oom_mutex);
1450         mem_cgroup_oom_unlock(mem);
1451         memcg_wakeup_oom(mem);
1452         mutex_unlock(&memcg_oom_mutex);
1453
1454         if (test_thread_flag(TIF_MEMDIE) || fatal_signal_pending(current))
1455                 return false;
1456         /* Give chance to dying process */
1457         schedule_timeout(1);
1458         return true;
1459 }
1460
1461 /*
1462  * Currently used to update mapped file statistics, but the routine can be
1463  * generalized to update other statistics as well.
1464  */
1465 void mem_cgroup_update_file_mapped(struct page *page, int val)
1466 {
1467         struct mem_cgroup *mem;
1468         struct page_cgroup *pc;
1469
1470         pc = lookup_page_cgroup(page);
1471         if (unlikely(!pc))
1472                 return;
1473
1474         lock_page_cgroup(pc);
1475         mem = pc->mem_cgroup;
1476         if (!mem || !PageCgroupUsed(pc))
1477                 goto done;
1478
1479         /*
1480          * Preemption is already disabled. We can use __this_cpu_xxx
1481          */
1482         if (val > 0) {
1483                 __this_cpu_inc(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1484                 SetPageCgroupFileMapped(pc);
1485         } else {
1486                 __this_cpu_dec(mem->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1487                 ClearPageCgroupFileMapped(pc);
1488         }
1489
1490 done:
1491         unlock_page_cgroup(pc);
1492 }
1493
1494 /*
1495  * size of first charge trial. "32" comes from vmscan.c's magic value.
1496  * TODO: maybe necessary to use big numbers in big irons.
1497  */
1498 #define CHARGE_SIZE     (32 * PAGE_SIZE)
1499 struct memcg_stock_pcp {
1500         struct mem_cgroup *cached; /* this never be root cgroup */
1501         int charge;
1502         struct work_struct work;
1503 };
1504 static DEFINE_PER_CPU(struct memcg_stock_pcp, memcg_stock);
1505 static atomic_t memcg_drain_count;
1506
1507 /*
1508  * Try to consume stocked charge on this cpu. If success, PAGE_SIZE is consumed
1509  * from local stock and true is returned. If the stock is 0 or charges from a
1510  * cgroup which is not current target, returns false. This stock will be
1511  * refilled.
1512  */
1513 static bool consume_stock(struct mem_cgroup *mem)
1514 {
1515         struct memcg_stock_pcp *stock;
1516         bool ret = true;
1517
1518         stock = &get_cpu_var(memcg_stock);
1519         if (mem == stock->cached && stock->charge)
1520                 stock->charge -= PAGE_SIZE;
1521         else /* need to call res_counter_charge */
1522                 ret = false;
1523         put_cpu_var(memcg_stock);
1524         return ret;
1525 }
1526
1527 /*
1528  * Returns stocks cached in percpu to res_counter and reset cached information.
1529  */
1530 static void drain_stock(struct memcg_stock_pcp *stock)
1531 {
1532         struct mem_cgroup *old = stock->cached;
1533
1534         if (stock->charge) {
1535                 res_counter_uncharge(&old->res, stock->charge);
1536                 if (do_swap_account)
1537                         res_counter_uncharge(&old->memsw, stock->charge);
1538         }
1539         stock->cached = NULL;
1540         stock->charge = 0;
1541 }
1542
1543 /*
1544  * This must be called under preempt disabled or must be called by
1545  * a thread which is pinned to local cpu.
1546  */
1547 static void drain_local_stock(struct work_struct *dummy)
1548 {
1549         struct memcg_stock_pcp *stock = &__get_cpu_var(memcg_stock);
1550         drain_stock(stock);
1551 }
1552
1553 /*
1554  * Cache charges(val) which is from res_counter, to local per_cpu area.
1555  * This will be consumed by consume_stock() function, later.
1556  */
1557 static void refill_stock(struct mem_cgroup *mem, int val)
1558 {
1559         struct memcg_stock_pcp *stock = &get_cpu_var(memcg_stock);
1560
1561         if (stock->cached != mem) { /* reset if necessary */
1562                 drain_stock(stock);
1563                 stock->cached = mem;
1564         }
1565         stock->charge += val;
1566         put_cpu_var(memcg_stock);
1567 }
1568
1569 /*
1570  * Tries to drain stocked charges in other cpus. This function is asynchronous
1571  * and just put a work per cpu for draining localy on each cpu. Caller can
1572  * expects some charges will be back to res_counter later but cannot wait for
1573  * it.
1574  */
1575 static void drain_all_stock_async(void)
1576 {
1577         int cpu;
1578         /* This function is for scheduling "drain" in asynchronous way.
1579          * The result of "drain" is not directly handled by callers. Then,
1580          * if someone is calling drain, we don't have to call drain more.
1581          * Anyway, WORK_STRUCT_PENDING check in queue_work_on() will catch if
1582          * there is a race. We just do loose check here.
1583          */
1584         if (atomic_read(&memcg_drain_count))
1585                 return;
1586         /* Notify other cpus that system-wide "drain" is running */
1587         atomic_inc(&memcg_drain_count);
1588         get_online_cpus();
1589         for_each_online_cpu(cpu) {
1590                 struct memcg_stock_pcp *stock = &per_cpu(memcg_stock, cpu);
1591                 schedule_work_on(cpu, &stock->work);
1592         }
1593         put_online_cpus();
1594         atomic_dec(&memcg_drain_count);
1595         /* We don't wait for flush_work */
1596 }
1597
1598 /* This is a synchronous drain interface. */
1599 static void drain_all_stock_sync(void)
1600 {
1601         /* called when force_empty is called */
1602         atomic_inc(&memcg_drain_count);
1603         schedule_on_each_cpu(drain_local_stock);
1604         atomic_dec(&memcg_drain_count);
1605 }
1606
1607 static int __cpuinit memcg_stock_cpu_callback(struct notifier_block *nb,
1608                                         unsigned long action,
1609                                         void *hcpu)
1610 {
1611         int cpu = (unsigned long)hcpu;
1612         struct memcg_stock_pcp *stock;
1613
1614         if (action != CPU_DEAD)
1615                 return NOTIFY_OK;
1616         stock = &per_cpu(memcg_stock, cpu);
1617         drain_stock(stock);
1618         return NOTIFY_OK;
1619 }
1620
1621
1622 /* See __mem_cgroup_try_charge() for details */
1623 enum {
1624         CHARGE_OK,              /* success */
1625         CHARGE_RETRY,           /* need to retry but retry is not bad */
1626         CHARGE_NOMEM,           /* we can't do more. return -ENOMEM */
1627         CHARGE_WOULDBLOCK,      /* GFP_WAIT wasn't set and no enough res. */
1628         CHARGE_OOM_DIE,         /* the current is killed because of OOM */
1629 };
1630
1631 static int __mem_cgroup_do_charge(struct mem_cgroup *mem, gfp_t gfp_mask,
1632                                 int csize, bool oom_check)
1633 {
1634         struct mem_cgroup *mem_over_limit;
1635         struct res_counter *fail_res;
1636         unsigned long flags = 0;
1637         int ret;
1638
1639         ret = res_counter_charge(&mem->res, csize, &fail_res);
1640
1641         if (likely(!ret)) {
1642                 if (!do_swap_account)
1643                         return CHARGE_OK;
1644                 ret = res_counter_charge(&mem->memsw, csize, &fail_res);
1645                 if (likely(!ret))
1646                         return CHARGE_OK;
1647
1648                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, memsw);
1649                 flags |= MEM_CGROUP_RECLAIM_NOSWAP;
1650         } else
1651                 mem_over_limit = mem_cgroup_from_res_counter(fail_res, res);
1652
1653         if (csize > PAGE_SIZE) /* change csize and retry */
1654                 return CHARGE_RETRY;
1655
1656         if (!(gfp_mask & __GFP_WAIT))
1657                 return CHARGE_WOULDBLOCK;
1658
1659         ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, NULL,
1660                                         gfp_mask, flags);
1661         /*
1662          * try_to_free_mem_cgroup_pages() might not give us a full
1663          * picture of reclaim. Some pages are reclaimed and might be
1664          * moved to swap cache or just unmapped from the cgroup.
1665          * Check the limit again to see if the reclaim reduced the
1666          * current usage of the cgroup before giving up
1667          */
1668         if (ret || mem_cgroup_check_under_limit(mem_over_limit))
1669                 return CHARGE_RETRY;
1670
1671         /*
1672          * At task move, charge accounts can be doubly counted. So, it's
1673          * better to wait until the end of task_move if something is going on.
1674          */
1675         if (mem_cgroup_wait_acct_move(mem_over_limit))
1676                 return CHARGE_RETRY;
1677
1678         /* If we don't need to call oom-killer at el, return immediately */
1679         if (!oom_check)
1680                 return CHARGE_NOMEM;
1681         /* check OOM */
1682         if (!mem_cgroup_handle_oom(mem_over_limit, gfp_mask))
1683                 return CHARGE_OOM_DIE;
1684
1685         return CHARGE_RETRY;
1686 }
1687
1688 /*
1689  * Unlike exported interface, "oom" parameter is added. if oom==true,
1690  * oom-killer can be invoked.
1691  */
1692 static int __mem_cgroup_try_charge(struct mm_struct *mm,
1693                 gfp_t gfp_mask, struct mem_cgroup **memcg, bool oom)
1694 {
1695         int nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1696         struct mem_cgroup *mem = NULL;
1697         int ret;
1698         int csize = CHARGE_SIZE;
1699
1700         /*
1701          * Unlike gloval-vm's OOM-kill, we're not in memory shortage
1702          * in system level. So, allow to go ahead dying process in addition to
1703          * MEMDIE process.
1704          */
1705         if (unlikely(test_thread_flag(TIF_MEMDIE)
1706                      || fatal_signal_pending(current)))
1707                 goto bypass;
1708
1709         /*
1710          * We always charge the cgroup the mm_struct belongs to.
1711          * The mm_struct's mem_cgroup changes on task migration if the
1712          * thread group leader migrates. It's possible that mm is not
1713          * set, if so charge the init_mm (happens for pagecache usage).
1714          */
1715         if (*memcg) {
1716                 mem = *memcg;
1717                 css_get(&mem->css);
1718         } else {
1719                 mem = try_get_mem_cgroup_from_mm(mm);
1720                 if (unlikely(!mem))
1721                         return 0;
1722                 *memcg = mem;
1723         }
1724
1725         VM_BUG_ON(css_is_removed(&mem->css));
1726         if (mem_cgroup_is_root(mem))
1727                 goto done;
1728
1729         do {
1730                 bool oom_check;
1731
1732                 if (consume_stock(mem))
1733                         goto done; /* don't need to fill stock */
1734                 /* If killed, bypass charge */
1735                 if (fatal_signal_pending(current))
1736                         goto bypass;
1737
1738                 oom_check = false;
1739                 if (oom && !nr_oom_retries) {
1740                         oom_check = true;
1741                         nr_oom_retries = MEM_CGROUP_RECLAIM_RETRIES;
1742                 }
1743
1744                 ret = __mem_cgroup_do_charge(mem, gfp_mask, csize, oom_check);
1745
1746                 switch (ret) {
1747                 case CHARGE_OK:
1748                         break;
1749                 case CHARGE_RETRY: /* not in OOM situation but retry */
1750                         csize = PAGE_SIZE;
1751                         break;
1752                 case CHARGE_WOULDBLOCK: /* !__GFP_WAIT */
1753                         goto nomem;
1754                 case CHARGE_NOMEM: /* OOM routine works */
1755                         if (!oom)
1756                                 goto nomem;
1757                         /* If oom, we never return -ENOMEM */
1758                         nr_oom_retries--;
1759                         break;
1760                 case CHARGE_OOM_DIE: /* Killed by OOM Killer */
1761                         goto bypass;
1762                 }
1763         } while (ret != CHARGE_OK);
1764
1765         if (csize > PAGE_SIZE)
1766                 refill_stock(mem, csize - PAGE_SIZE);
1767 done:
1768         return 0;
1769 nomem:
1770         css_put(&mem->css);
1771         return -ENOMEM;
1772 bypass:
1773         if (mem)
1774                 css_put(&mem->css);
1775         *memcg = NULL;
1776         return 0;
1777 }
1778
1779 /*
1780  * Somemtimes we have to undo a charge we got by try_charge().
1781  * This function is for that and do uncharge, put css's refcnt.
1782  * gotten by try_charge().
1783  */
1784 static void __mem_cgroup_cancel_charge(struct mem_cgroup *mem,
1785                                                         unsigned long count)
1786 {
1787         if (!mem_cgroup_is_root(mem)) {
1788                 res_counter_uncharge(&mem->res, PAGE_SIZE * count);
1789                 if (do_swap_account)
1790                         res_counter_uncharge(&mem->memsw, PAGE_SIZE * count);
1791                 VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
1792                 WARN_ON_ONCE(count > INT_MAX);
1793                 __css_put(&mem->css, (int)count);
1794         }
1795         /* we don't need css_put for root */
1796 }
1797
1798 static void mem_cgroup_cancel_charge(struct mem_cgroup *mem)
1799 {
1800         __mem_cgroup_cancel_charge(mem, 1);
1801 }
1802
1803 /*
1804  * A helper function to get mem_cgroup from ID. must be called under
1805  * rcu_read_lock(). The caller must check css_is_removed() or some if
1806  * it's concern. (dropping refcnt from swap can be called against removed
1807  * memcg.)
1808  */
1809 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
1810 {
1811         struct cgroup_subsys_state *css;
1812
1813         /* ID 0 is unused ID */
1814         if (!id)
1815                 return NULL;
1816         css = css_lookup(&mem_cgroup_subsys, id);
1817         if (!css)
1818                 return NULL;
1819         return container_of(css, struct mem_cgroup, css);
1820 }
1821
1822 struct mem_cgroup *try_get_mem_cgroup_from_page(struct page *page)
1823 {
1824         struct mem_cgroup *mem = NULL;
1825         struct page_cgroup *pc;
1826         unsigned short id;
1827         swp_entry_t ent;
1828
1829         VM_BUG_ON(!PageLocked(page));
1830
1831         pc = lookup_page_cgroup(page);
1832         lock_page_cgroup(pc);
1833         if (PageCgroupUsed(pc)) {
1834                 mem = pc->mem_cgroup;
1835                 if (mem && !css_tryget(&mem->css))
1836                         mem = NULL;
1837         } else if (PageSwapCache(page)) {
1838                 ent.val = page_private(page);
1839                 id = lookup_swap_cgroup(ent);
1840                 rcu_read_lock();
1841                 mem = mem_cgroup_lookup(id);
1842                 if (mem && !css_tryget(&mem->css))
1843                         mem = NULL;
1844                 rcu_read_unlock();
1845         }
1846         unlock_page_cgroup(pc);
1847         return mem;
1848 }
1849
1850 /*
1851  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
1852  * USED state. If already USED, uncharge and return.
1853  */
1854
1855 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
1856                                      struct page_cgroup *pc,
1857                                      enum charge_type ctype)
1858 {
1859         /* try_charge() can return NULL to *memcg, taking care of it. */
1860         if (!mem)
1861                 return;
1862
1863         lock_page_cgroup(pc);
1864         if (unlikely(PageCgroupUsed(pc))) {
1865                 unlock_page_cgroup(pc);
1866                 mem_cgroup_cancel_charge(mem);
1867                 return;
1868         }
1869
1870         pc->mem_cgroup = mem;
1871         /*
1872          * We access a page_cgroup asynchronously without lock_page_cgroup().
1873          * Especially when a page_cgroup is taken from a page, pc->mem_cgroup
1874          * is accessed after testing USED bit. To make pc->mem_cgroup visible
1875          * before USED bit, we need memory barrier here.
1876          * See mem_cgroup_add_lru_list(), etc.
1877          */
1878         smp_wmb();
1879         switch (ctype) {
1880         case MEM_CGROUP_CHARGE_TYPE_CACHE:
1881         case MEM_CGROUP_CHARGE_TYPE_SHMEM:
1882                 SetPageCgroupCache(pc);
1883                 SetPageCgroupUsed(pc);
1884                 break;
1885         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
1886                 ClearPageCgroupCache(pc);
1887                 SetPageCgroupUsed(pc);
1888                 break;
1889         default:
1890                 break;
1891         }
1892
1893         mem_cgroup_charge_statistics(mem, pc, true);
1894
1895         unlock_page_cgroup(pc);
1896         /*
1897          * "charge_statistics" updated event counter. Then, check it.
1898          * Insert ancestor (and ancestor's ancestors), to softlimit RB-tree.
1899          * if they exceeds softlimit.
1900          */
1901         memcg_check_events(mem, pc->page);
1902 }
1903
1904 /**
1905  * __mem_cgroup_move_account - move account of the page
1906  * @pc: page_cgroup of the page.
1907  * @from: mem_cgroup which the page is moved from.
1908  * @to: mem_cgroup which the page is moved to. @from != @to.
1909  * @uncharge: whether we should call uncharge and css_put against @from.
1910  *
1911  * The caller must confirm following.
1912  * - page is not on LRU (isolate_page() is useful.)
1913  * - the pc is locked, used, and ->mem_cgroup points to @from.
1914  *
1915  * This function doesn't do "charge" nor css_get to new cgroup. It should be
1916  * done by a caller(__mem_cgroup_try_charge would be usefull). If @uncharge is
1917  * true, this function does "uncharge" from old cgroup, but it doesn't if
1918  * @uncharge is false, so a caller should do "uncharge".
1919  */
1920
1921 static void __mem_cgroup_move_account(struct page_cgroup *pc,
1922         struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
1923 {
1924         VM_BUG_ON(from == to);
1925         VM_BUG_ON(PageLRU(pc->page));
1926         VM_BUG_ON(!PageCgroupLocked(pc));
1927         VM_BUG_ON(!PageCgroupUsed(pc));
1928         VM_BUG_ON(pc->mem_cgroup != from);
1929
1930         if (PageCgroupFileMapped(pc)) {
1931                 /* Update mapped_file data for mem_cgroup */
1932                 preempt_disable();
1933                 __this_cpu_dec(from->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1934                 __this_cpu_inc(to->stat->count[MEM_CGROUP_STAT_FILE_MAPPED]);
1935                 preempt_enable();
1936         }
1937         mem_cgroup_charge_statistics(from, pc, false);
1938         if (uncharge)
1939                 /* This is not "cancel", but cancel_charge does all we need. */
1940                 mem_cgroup_cancel_charge(from);
1941
1942         /* caller should have done css_get */
1943         pc->mem_cgroup = to;
1944         mem_cgroup_charge_statistics(to, pc, true);
1945         /*
1946          * We charges against "to" which may not have any tasks. Then, "to"
1947          * can be under rmdir(). But in current implementation, caller of
1948          * this function is just force_empty() and move charge, so it's
1949          * garanteed that "to" is never removed. So, we don't check rmdir
1950          * status here.
1951          */
1952 }
1953
1954 /*
1955  * check whether the @pc is valid for moving account and call
1956  * __mem_cgroup_move_account()
1957  */
1958 static int mem_cgroup_move_account(struct page_cgroup *pc,
1959                 struct mem_cgroup *from, struct mem_cgroup *to, bool uncharge)
1960 {
1961         int ret = -EINVAL;
1962         lock_page_cgroup(pc);
1963         if (PageCgroupUsed(pc) && pc->mem_cgroup == from) {
1964                 __mem_cgroup_move_account(pc, from, to, uncharge);
1965                 ret = 0;
1966         }
1967         unlock_page_cgroup(pc);
1968         /*
1969          * check events
1970          */
1971         memcg_check_events(to, pc->page);
1972         memcg_check_events(from, pc->page);
1973         return ret;
1974 }
1975
1976 /*
1977  * move charges to its parent.
1978  */
1979
1980 static int mem_cgroup_move_parent(struct page_cgroup *pc,
1981                                   struct mem_cgroup *child,
1982                                   gfp_t gfp_mask)
1983 {
1984         struct page *page = pc->page;
1985         struct cgroup *cg = child->css.cgroup;
1986         struct cgroup *pcg = cg->parent;
1987         struct mem_cgroup *parent;
1988         int ret;
1989
1990         /* Is ROOT ? */
1991         if (!pcg)
1992                 return -EINVAL;
1993
1994         ret = -EBUSY;
1995         if (!get_page_unless_zero(page))
1996                 goto out;
1997         if (isolate_lru_page(page))
1998                 goto put;
1999
2000         parent = mem_cgroup_from_cont(pcg);
2001         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
2002         if (ret || !parent)
2003                 goto put_back;
2004
2005         ret = mem_cgroup_move_account(pc, child, parent, true);
2006         if (ret)
2007                 mem_cgroup_cancel_charge(parent);
2008 put_back:
2009         putback_lru_page(page);
2010 put:
2011         put_page(page);
2012 out:
2013         return ret;
2014 }
2015
2016 /*
2017  * Charge the memory controller for page usage.
2018  * Return
2019  * 0 if the charge was successful
2020  * < 0 if the cgroup is over its limit
2021  */
2022 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
2023                                 gfp_t gfp_mask, enum charge_type ctype,
2024                                 struct mem_cgroup *memcg)
2025 {
2026         struct mem_cgroup *mem;
2027         struct page_cgroup *pc;
2028         int ret;
2029
2030         pc = lookup_page_cgroup(page);
2031         /* can happen at boot */
2032         if (unlikely(!pc))
2033                 return 0;
2034         prefetchw(pc);
2035
2036         mem = memcg;
2037         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
2038         if (ret || !mem)
2039                 return ret;
2040
2041         __mem_cgroup_commit_charge(mem, pc, ctype);
2042         return 0;
2043 }
2044
2045 int mem_cgroup_newpage_charge(struct page *page,
2046                               struct mm_struct *mm, gfp_t gfp_mask)
2047 {
2048         if (mem_cgroup_disabled())
2049                 return 0;
2050         if (PageCompound(page))
2051                 return 0;
2052         /*
2053          * If already mapped, we don't have to account.
2054          * If page cache, page->mapping has address_space.
2055          * But page->mapping may have out-of-use anon_vma pointer,
2056          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
2057          * is NULL.
2058          */
2059         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
2060                 return 0;
2061         if (unlikely(!mm))
2062                 mm = &init_mm;
2063         return mem_cgroup_charge_common(page, mm, gfp_mask,
2064                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
2065 }
2066
2067 static void
2068 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2069                                         enum charge_type ctype);
2070
2071 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
2072                                 gfp_t gfp_mask)
2073 {
2074         struct mem_cgroup *mem = NULL;
2075         int ret;
2076
2077         if (mem_cgroup_disabled())
2078                 return 0;
2079         if (PageCompound(page))
2080                 return 0;
2081         /*
2082          * Corner case handling. This is called from add_to_page_cache()
2083          * in usual. But some FS (shmem) precharges this page before calling it
2084          * and call add_to_page_cache() with GFP_NOWAIT.
2085          *
2086          * For GFP_NOWAIT case, the page may be pre-charged before calling
2087          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
2088          * charge twice. (It works but has to pay a bit larger cost.)
2089          * And when the page is SwapCache, it should take swap information
2090          * into account. This is under lock_page() now.
2091          */
2092         if (!(gfp_mask & __GFP_WAIT)) {
2093                 struct page_cgroup *pc;
2094
2095
2096                 pc = lookup_page_cgroup(page);
2097                 if (!pc)
2098                         return 0;
2099                 lock_page_cgroup(pc);
2100                 if (PageCgroupUsed(pc)) {
2101                         unlock_page_cgroup(pc);
2102                         return 0;
2103                 }
2104                 unlock_page_cgroup(pc);
2105         }
2106
2107         if (unlikely(!mm && !mem))
2108                 mm = &init_mm;
2109
2110         if (page_is_file_cache(page))
2111                 return mem_cgroup_charge_common(page, mm, gfp_mask,
2112                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
2113
2114         /* shmem */
2115         if (PageSwapCache(page)) {
2116                 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2117                 if (!ret)
2118                         __mem_cgroup_commit_charge_swapin(page, mem,
2119                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
2120         } else
2121                 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
2122                                         MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
2123
2124         return ret;
2125 }
2126
2127 /*
2128  * While swap-in, try_charge -> commit or cancel, the page is locked.
2129  * And when try_charge() successfully returns, one refcnt to memcg without
2130  * struct page_cgroup is acquired. This refcnt will be consumed by
2131  * "commit()" or removed by "cancel()"
2132  */
2133 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
2134                                  struct page *page,
2135                                  gfp_t mask, struct mem_cgroup **ptr)
2136 {
2137         struct mem_cgroup *mem;
2138         int ret;
2139
2140         if (mem_cgroup_disabled())
2141                 return 0;
2142
2143         if (!do_swap_account)
2144                 goto charge_cur_mm;
2145         /*
2146          * A racing thread's fault, or swapoff, may have already updated
2147          * the pte, and even removed page from swap cache: in those cases
2148          * do_swap_page()'s pte_same() test will fail; but there's also a
2149          * KSM case which does need to charge the page.
2150          */
2151         if (!PageSwapCache(page))
2152                 goto charge_cur_mm;
2153         mem = try_get_mem_cgroup_from_page(page);
2154         if (!mem)
2155                 goto charge_cur_mm;
2156         *ptr = mem;
2157         ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
2158         /* drop extra refcnt from tryget */
2159         css_put(&mem->css);
2160         return ret;
2161 charge_cur_mm:
2162         if (unlikely(!mm))
2163                 mm = &init_mm;
2164         return __mem_cgroup_try_charge(mm, mask, ptr, true);
2165 }
2166
2167 static void
2168 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
2169                                         enum charge_type ctype)
2170 {
2171         struct page_cgroup *pc;
2172
2173         if (mem_cgroup_disabled())
2174                 return;
2175         if (!ptr)
2176                 return;
2177         cgroup_exclude_rmdir(&ptr->css);
2178         pc = lookup_page_cgroup(page);
2179         mem_cgroup_lru_del_before_commit_swapcache(page);
2180         __mem_cgroup_commit_charge(ptr, pc, ctype);
2181         mem_cgroup_lru_add_after_commit_swapcache(page);
2182         /*
2183          * Now swap is on-memory. This means this page may be
2184          * counted both as mem and swap....double count.
2185          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
2186          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
2187          * may call delete_from_swap_cache() before reach here.
2188          */
2189         if (do_swap_account && PageSwapCache(page)) {
2190                 swp_entry_t ent = {.val = page_private(page)};
2191                 unsigned short id;
2192                 struct mem_cgroup *memcg;
2193
2194                 id = swap_cgroup_record(ent, 0);
2195                 rcu_read_lock();
2196                 memcg = mem_cgroup_lookup(id);
2197                 if (memcg) {
2198                         /*
2199                          * This recorded memcg can be obsolete one. So, avoid
2200                          * calling css_tryget
2201                          */
2202                         if (!mem_cgroup_is_root(memcg))
2203                                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2204                         mem_cgroup_swap_statistics(memcg, false);
2205                         mem_cgroup_put(memcg);
2206                 }
2207                 rcu_read_unlock();
2208         }
2209         /*
2210          * At swapin, we may charge account against cgroup which has no tasks.
2211          * So, rmdir()->pre_destroy() can be called while we do this charge.
2212          * In that case, we need to call pre_destroy() again. check it here.
2213          */
2214         cgroup_release_and_wakeup_rmdir(&ptr->css);
2215 }
2216
2217 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
2218 {
2219         __mem_cgroup_commit_charge_swapin(page, ptr,
2220                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
2221 }
2222
2223 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
2224 {
2225         if (mem_cgroup_disabled())
2226                 return;
2227         if (!mem)
2228                 return;
2229         mem_cgroup_cancel_charge(mem);
2230 }
2231
2232 static void
2233 __do_uncharge(struct mem_cgroup *mem, const enum charge_type ctype)
2234 {
2235         struct memcg_batch_info *batch = NULL;
2236         bool uncharge_memsw = true;
2237         /* If swapout, usage of swap doesn't decrease */
2238         if (!do_swap_account || ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2239                 uncharge_memsw = false;
2240
2241         batch = &current->memcg_batch;
2242         /*
2243          * In usual, we do css_get() when we remember memcg pointer.
2244          * But in this case, we keep res->usage until end of a series of
2245          * uncharges. Then, it's ok to ignore memcg's refcnt.
2246          */
2247         if (!batch->memcg)
2248                 batch->memcg = mem;
2249         /*
2250          * do_batch > 0 when unmapping pages or inode invalidate/truncate.
2251          * In those cases, all pages freed continously can be expected to be in
2252          * the same cgroup and we have chance to coalesce uncharges.
2253          * But we do uncharge one by one if this is killed by OOM(TIF_MEMDIE)
2254          * because we want to do uncharge as soon as possible.
2255          */
2256
2257         if (!batch->do_batch || test_thread_flag(TIF_MEMDIE))
2258                 goto direct_uncharge;
2259
2260         /*
2261          * In typical case, batch->memcg == mem. This means we can
2262          * merge a series of uncharges to an uncharge of res_counter.
2263          * If not, we uncharge res_counter ony by one.
2264          */
2265         if (batch->memcg != mem)
2266                 goto direct_uncharge;
2267         /* remember freed charge and uncharge it later */
2268         batch->bytes += PAGE_SIZE;
2269         if (uncharge_memsw)
2270                 batch->memsw_bytes += PAGE_SIZE;
2271         return;
2272 direct_uncharge:
2273         res_counter_uncharge(&mem->res, PAGE_SIZE);
2274         if (uncharge_memsw)
2275                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
2276         if (unlikely(batch->memcg != mem))
2277                 memcg_oom_recover(mem);
2278         return;
2279 }
2280
2281 /*
2282  * uncharge if !page_mapped(page)
2283  */
2284 static struct mem_cgroup *
2285 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
2286 {
2287         struct page_cgroup *pc;
2288         struct mem_cgroup *mem = NULL;
2289         struct mem_cgroup_per_zone *mz;
2290
2291         if (mem_cgroup_disabled())
2292                 return NULL;
2293
2294         if (PageSwapCache(page))
2295                 return NULL;
2296
2297         /*
2298          * Check if our page_cgroup is valid
2299          */
2300         pc = lookup_page_cgroup(page);
2301         if (unlikely(!pc || !PageCgroupUsed(pc)))
2302                 return NULL;
2303
2304         lock_page_cgroup(pc);
2305
2306         mem = pc->mem_cgroup;
2307
2308         if (!PageCgroupUsed(pc))
2309                 goto unlock_out;
2310
2311         switch (ctype) {
2312         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
2313         case MEM_CGROUP_CHARGE_TYPE_DROP:
2314                 /* See mem_cgroup_prepare_migration() */
2315                 if (page_mapped(page) || PageCgroupMigration(pc))
2316                         goto unlock_out;
2317                 break;
2318         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
2319                 if (!PageAnon(page)) {  /* Shared memory */
2320                         if (page->mapping && !page_is_file_cache(page))
2321                                 goto unlock_out;
2322                 } else if (page_mapped(page)) /* Anon */
2323                                 goto unlock_out;
2324                 break;
2325         default:
2326                 break;
2327         }
2328
2329         if (!mem_cgroup_is_root(mem))
2330                 __do_uncharge(mem, ctype);
2331         if (ctype == MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2332                 mem_cgroup_swap_statistics(mem, true);
2333         mem_cgroup_charge_statistics(mem, pc, false);
2334
2335         ClearPageCgroupUsed(pc);
2336         /*
2337          * pc->mem_cgroup is not cleared here. It will be accessed when it's
2338          * freed from LRU. This is safe because uncharged page is expected not
2339          * to be reused (freed soon). Exception is SwapCache, it's handled by
2340          * special functions.
2341          */
2342
2343         mz = page_cgroup_zoneinfo(pc);
2344         unlock_page_cgroup(pc);
2345
2346         memcg_check_events(mem, page);
2347         /* at swapout, this memcg will be accessed to record to swap */
2348         if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
2349                 css_put(&mem->css);
2350
2351         return mem;
2352
2353 unlock_out:
2354         unlock_page_cgroup(pc);
2355         return NULL;
2356 }
2357
2358 void mem_cgroup_uncharge_page(struct page *page)
2359 {
2360         /* early check. */
2361         if (page_mapped(page))
2362                 return;
2363         if (page->mapping && !PageAnon(page))
2364                 return;
2365         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
2366 }
2367
2368 void mem_cgroup_uncharge_cache_page(struct page *page)
2369 {
2370         VM_BUG_ON(page_mapped(page));
2371         VM_BUG_ON(page->mapping);
2372         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
2373 }
2374
2375 /*
2376  * Batch_start/batch_end is called in unmap_page_range/invlidate/trucate.
2377  * In that cases, pages are freed continuously and we can expect pages
2378  * are in the same memcg. All these calls itself limits the number of
2379  * pages freed at once, then uncharge_start/end() is called properly.
2380  * This may be called prural(2) times in a context,
2381  */
2382
2383 void mem_cgroup_uncharge_start(void)
2384 {
2385         current->memcg_batch.do_batch++;
2386         /* We can do nest. */
2387         if (current->memcg_batch.do_batch == 1) {
2388                 current->memcg_batch.memcg = NULL;
2389                 current->memcg_batch.bytes = 0;
2390                 current->memcg_batch.memsw_bytes = 0;
2391         }
2392 }
2393
2394 void mem_cgroup_uncharge_end(void)
2395 {
2396         struct memcg_batch_info *batch = &current->memcg_batch;
2397
2398         if (!batch->do_batch)
2399                 return;
2400
2401         batch->do_batch--;
2402         if (batch->do_batch) /* If stacked, do nothing. */
2403                 return;
2404
2405         if (!batch->memcg)
2406                 return;
2407         /*
2408          * This "batch->memcg" is valid without any css_get/put etc...
2409          * bacause we hide charges behind us.
2410          */
2411         if (batch->bytes)
2412                 res_counter_uncharge(&batch->memcg->res, batch->bytes);
2413         if (batch->memsw_bytes)
2414                 res_counter_uncharge(&batch->memcg->memsw, batch->memsw_bytes);
2415         memcg_oom_recover(batch->memcg);
2416         /* forget this pointer (for sanity check) */
2417         batch->memcg = NULL;
2418 }
2419
2420 #ifdef CONFIG_SWAP
2421 /*
2422  * called after __delete_from_swap_cache() and drop "page" account.
2423  * memcg information is recorded to swap_cgroup of "ent"
2424  */
2425 void
2426 mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
2427 {
2428         struct mem_cgroup *memcg;
2429         int ctype = MEM_CGROUP_CHARGE_TYPE_SWAPOUT;
2430
2431         if (!swapout) /* this was a swap cache but the swap is unused ! */
2432                 ctype = MEM_CGROUP_CHARGE_TYPE_DROP;
2433
2434         memcg = __mem_cgroup_uncharge_common(page, ctype);
2435
2436         /* record memcg information */
2437         if (do_swap_account && swapout && memcg) {
2438                 swap_cgroup_record(ent, css_id(&memcg->css));
2439                 mem_cgroup_get(memcg);
2440         }
2441         if (swapout && memcg)
2442                 css_put(&memcg->css);
2443 }
2444 #endif
2445
2446 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2447 /*
2448  * called from swap_entry_free(). remove record in swap_cgroup and
2449  * uncharge "memsw" account.
2450  */
2451 void mem_cgroup_uncharge_swap(swp_entry_t ent)
2452 {
2453         struct mem_cgroup *memcg;
2454         unsigned short id;
2455
2456         if (!do_swap_account)
2457                 return;
2458
2459         id = swap_cgroup_record(ent, 0);
2460         rcu_read_lock();
2461         memcg = mem_cgroup_lookup(id);
2462         if (memcg) {
2463                 /*
2464                  * We uncharge this because swap is freed.
2465                  * This memcg can be obsolete one. We avoid calling css_tryget
2466                  */
2467                 if (!mem_cgroup_is_root(memcg))
2468                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
2469                 mem_cgroup_swap_statistics(memcg, false);
2470                 mem_cgroup_put(memcg);
2471         }
2472         rcu_read_unlock();
2473 }
2474
2475 /**
2476  * mem_cgroup_move_swap_account - move swap charge and swap_cgroup's record.
2477  * @entry: swap entry to be moved
2478  * @from:  mem_cgroup which the entry is moved from
2479  * @to:  mem_cgroup which the entry is moved to
2480  * @need_fixup: whether we should fixup res_counters and refcounts.
2481  *
2482  * It succeeds only when the swap_cgroup's record for this entry is the same
2483  * as the mem_cgroup's id of @from.
2484  *
2485  * Returns 0 on success, -EINVAL on failure.
2486  *
2487  * The caller must have charged to @to, IOW, called res_counter_charge() about
2488  * both res and memsw, and called css_get().
2489  */
2490 static int mem_cgroup_move_swap_account(swp_entry_t entry,
2491                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2492 {
2493         unsigned short old_id, new_id;
2494
2495         old_id = css_id(&from->css);
2496         new_id = css_id(&to->css);
2497
2498         if (swap_cgroup_cmpxchg(entry, old_id, new_id) == old_id) {
2499                 mem_cgroup_swap_statistics(from, false);
2500                 mem_cgroup_swap_statistics(to, true);
2501                 /*
2502                  * This function is only called from task migration context now.
2503                  * It postpones res_counter and refcount handling till the end
2504                  * of task migration(mem_cgroup_clear_mc()) for performance
2505                  * improvement. But we cannot postpone mem_cgroup_get(to)
2506                  * because if the process that has been moved to @to does
2507                  * swap-in, the refcount of @to might be decreased to 0.
2508                  */
2509                 mem_cgroup_get(to);
2510                 if (need_fixup) {
2511                         if (!mem_cgroup_is_root(from))
2512                                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
2513                         mem_cgroup_put(from);
2514                         /*
2515                          * we charged both to->res and to->memsw, so we should
2516                          * uncharge to->res.
2517                          */
2518                         if (!mem_cgroup_is_root(to))
2519                                 res_counter_uncharge(&to->res, PAGE_SIZE);
2520                         css_put(&to->css);
2521                 }
2522                 return 0;
2523         }
2524         return -EINVAL;
2525 }
2526 #else
2527 static inline int mem_cgroup_move_swap_account(swp_entry_t entry,
2528                 struct mem_cgroup *from, struct mem_cgroup *to, bool need_fixup)
2529 {
2530         return -EINVAL;
2531 }
2532 #endif
2533
2534 /*
2535  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
2536  * page belongs to.
2537  */
2538 int mem_cgroup_prepare_migration(struct page *page,
2539         struct page *newpage, struct mem_cgroup **ptr)
2540 {
2541         struct page_cgroup *pc;
2542         struct mem_cgroup *mem = NULL;
2543         enum charge_type ctype;
2544         int ret = 0;
2545
2546         if (mem_cgroup_disabled())
2547                 return 0;
2548
2549         pc = lookup_page_cgroup(page);
2550         lock_page_cgroup(pc);
2551         if (PageCgroupUsed(pc)) {
2552                 mem = pc->mem_cgroup;
2553                 css_get(&mem->css);
2554                 /*
2555                  * At migrating an anonymous page, its mapcount goes down
2556                  * to 0 and uncharge() will be called. But, even if it's fully
2557                  * unmapped, migration may fail and this page has to be
2558                  * charged again. We set MIGRATION flag here and delay uncharge
2559                  * until end_migration() is called
2560                  *
2561                  * Corner Case Thinking
2562                  * A)
2563                  * When the old page was mapped as Anon and it's unmap-and-freed
2564                  * while migration was ongoing.
2565                  * If unmap finds the old page, uncharge() of it will be delayed
2566                  * until end_migration(). If unmap finds a new page, it's
2567                  * uncharged when it make mapcount to be 1->0. If unmap code
2568                  * finds swap_migration_entry, the new page will not be mapped
2569                  * and end_migration() will find it(mapcount==0).
2570                  *
2571                  * B)
2572                  * When the old page was mapped but migraion fails, the kernel
2573                  * remaps it. A charge for it is kept by MIGRATION flag even
2574                  * if mapcount goes down to 0. We can do remap successfully
2575                  * without charging it again.
2576                  *
2577                  * C)
2578                  * The "old" page is under lock_page() until the end of
2579                  * migration, so, the old page itself will not be swapped-out.
2580                  * If the new page is swapped out before end_migraton, our
2581                  * hook to usual swap-out path will catch the event.
2582                  */
2583                 if (PageAnon(page))
2584                         SetPageCgroupMigration(pc);
2585         }
2586         unlock_page_cgroup(pc);
2587         /*
2588          * If the page is not charged at this point,
2589          * we return here.
2590          */
2591         if (!mem)
2592                 return 0;
2593
2594         *ptr = mem;
2595         ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, ptr, false);
2596         css_put(&mem->css);/* drop extra refcnt */
2597         if (ret || *ptr == NULL) {
2598                 if (PageAnon(page)) {
2599                         lock_page_cgroup(pc);
2600                         ClearPageCgroupMigration(pc);
2601                         unlock_page_cgroup(pc);
2602                         /*
2603                          * The old page may be fully unmapped while we kept it.
2604                          */
2605                         mem_cgroup_uncharge_page(page);
2606                 }
2607                 return -ENOMEM;
2608         }
2609         /*
2610          * We charge new page before it's used/mapped. So, even if unlock_page()
2611          * is called before end_migration, we can catch all events on this new
2612          * page. In the case new page is migrated but not remapped, new page's
2613          * mapcount will be finally 0 and we call uncharge in end_migration().
2614          */
2615         pc = lookup_page_cgroup(newpage);
2616         if (PageAnon(page))
2617                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
2618         else if (page_is_file_cache(page))
2619                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
2620         else
2621                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
2622         __mem_cgroup_commit_charge(mem, pc, ctype);
2623         return ret;
2624 }
2625
2626 /* remove redundant charge if migration failed*/
2627 void mem_cgroup_end_migration(struct mem_cgroup *mem,
2628         struct page *oldpage, struct page *newpage)
2629 {
2630         struct page *used, *unused;
2631         struct page_cgroup *pc;
2632
2633         if (!mem)
2634                 return;
2635         /* blocks rmdir() */
2636         cgroup_exclude_rmdir(&mem->css);
2637         /* at migration success, oldpage->mapping is NULL. */
2638         if (oldpage->mapping) {
2639                 used = oldpage;
2640                 unused = newpage;
2641         } else {
2642                 used = newpage;
2643                 unused = oldpage;
2644         }
2645         /*
2646          * We disallowed uncharge of pages under migration because mapcount
2647          * of the page goes down to zero, temporarly.
2648          * Clear the flag and check the page should be charged.
2649          */
2650         pc = lookup_page_cgroup(oldpage);
2651         lock_page_cgroup(pc);
2652         ClearPageCgroupMigration(pc);
2653         unlock_page_cgroup(pc);
2654
2655         if (unused != oldpage)
2656                 pc = lookup_page_cgroup(unused);
2657         __mem_cgroup_uncharge_common(unused, MEM_CGROUP_CHARGE_TYPE_FORCE);
2658
2659         pc = lookup_page_cgroup(used);
2660         /*
2661          * If a page is a file cache, radix-tree replacement is very atomic
2662          * and we can skip this check. When it was an Anon page, its mapcount
2663          * goes down to 0. But because we added MIGRATION flage, it's not
2664          * uncharged yet. There are several case but page->mapcount check
2665          * and USED bit check in mem_cgroup_uncharge_page() will do enough
2666          * check. (see prepare_charge() also)
2667          */
2668         if (PageAnon(used))
2669                 mem_cgroup_uncharge_page(used);
2670         /*
2671          * At migration, we may charge account against cgroup which has no
2672          * tasks.
2673          * So, rmdir()->pre_destroy() can be called while we do this charge.
2674          * In that case, we need to call pre_destroy() again. check it here.
2675          */
2676         cgroup_release_and_wakeup_rmdir(&mem->css);
2677 }
2678
2679 /*
2680  * A call to try to shrink memory usage on charge failure at shmem's swapin.
2681  * Calling hierarchical_reclaim is not enough because we should update
2682  * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
2683  * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
2684  * not from the memcg which this page would be charged to.
2685  * try_charge_swapin does all of these works properly.
2686  */
2687 int mem_cgroup_shmem_charge_fallback(struct page *page,
2688                             struct mm_struct *mm,
2689                             gfp_t gfp_mask)
2690 {
2691         struct mem_cgroup *mem = NULL;
2692         int ret;
2693
2694         if (mem_cgroup_disabled())
2695                 return 0;
2696
2697         ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
2698         if (!ret)
2699                 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
2700
2701         return ret;
2702 }
2703
2704 static DEFINE_MUTEX(set_limit_mutex);
2705
2706 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
2707                                 unsigned long long val)
2708 {
2709         int retry_count;
2710         u64 memswlimit, memlimit;
2711         int ret = 0;
2712         int children = mem_cgroup_count_children(memcg);
2713         u64 curusage, oldusage;
2714         int enlarge;
2715
2716         /*
2717          * For keeping hierarchical_reclaim simple, how long we should retry
2718          * is depends on callers. We set our retry-count to be function
2719          * of # of children which we should visit in this loop.
2720          */
2721         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
2722
2723         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2724
2725         enlarge = 0;
2726         while (retry_count) {
2727                 if (signal_pending(current)) {
2728                         ret = -EINTR;
2729                         break;
2730                 }
2731                 /*
2732                  * Rather than hide all in some function, I do this in
2733                  * open coded manner. You see what this really does.
2734                  * We have to guarantee mem->res.limit < mem->memsw.limit.
2735                  */
2736                 mutex_lock(&set_limit_mutex);
2737                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2738                 if (memswlimit < val) {
2739                         ret = -EINVAL;
2740                         mutex_unlock(&set_limit_mutex);
2741                         break;
2742                 }
2743
2744                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2745                 if (memlimit < val)
2746                         enlarge = 1;
2747
2748                 ret = res_counter_set_limit(&memcg->res, val);
2749                 if (!ret) {
2750                         if (memswlimit == val)
2751                                 memcg->memsw_is_minimum = true;
2752                         else
2753                                 memcg->memsw_is_minimum = false;
2754                 }
2755                 mutex_unlock(&set_limit_mutex);
2756
2757                 if (!ret)
2758                         break;
2759
2760                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
2761                                                 MEM_CGROUP_RECLAIM_SHRINK);
2762                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
2763                 /* Usage is reduced ? */
2764                 if (curusage >= oldusage)
2765                         retry_count--;
2766                 else
2767                         oldusage = curusage;
2768         }
2769         if (!ret && enlarge)
2770                 memcg_oom_recover(memcg);
2771
2772         return ret;
2773 }
2774
2775 static int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
2776                                         unsigned long long val)
2777 {
2778         int retry_count;
2779         u64 memlimit, memswlimit, oldusage, curusage;
2780         int children = mem_cgroup_count_children(memcg);
2781         int ret = -EBUSY;
2782         int enlarge = 0;
2783
2784         /* see mem_cgroup_resize_res_limit */
2785         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
2786         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
2787         while (retry_count) {
2788                 if (signal_pending(current)) {
2789                         ret = -EINTR;
2790                         break;
2791                 }
2792                 /*
2793                  * Rather than hide all in some function, I do this in
2794                  * open coded manner. You see what this really does.
2795                  * We have to guarantee mem->res.limit < mem->memsw.limit.
2796                  */
2797                 mutex_lock(&set_limit_mutex);
2798                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
2799                 if (memlimit > val) {
2800                         ret = -EINVAL;
2801                         mutex_unlock(&set_limit_mutex);
2802                         break;
2803                 }
2804                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2805                 if (memswlimit < val)
2806                         enlarge = 1;
2807                 ret = res_counter_set_limit(&memcg->memsw, val);
2808                 if (!ret) {
2809                         if (memlimit == val)
2810                                 memcg->memsw_is_minimum = true;
2811                         else
2812                                 memcg->memsw_is_minimum = false;
2813                 }
2814                 mutex_unlock(&set_limit_mutex);
2815
2816                 if (!ret)
2817                         break;
2818
2819                 mem_cgroup_hierarchical_reclaim(memcg, NULL, GFP_KERNEL,
2820                                                 MEM_CGROUP_RECLAIM_NOSWAP |
2821                                                 MEM_CGROUP_RECLAIM_SHRINK);
2822                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
2823                 /* Usage is reduced ? */
2824                 if (curusage >= oldusage)
2825                         retry_count--;
2826                 else
2827                         oldusage = curusage;
2828         }
2829         if (!ret && enlarge)
2830                 memcg_oom_recover(memcg);
2831         return ret;
2832 }
2833
2834 unsigned long mem_cgroup_soft_limit_reclaim(struct zone *zone, int order,
2835                                                 gfp_t gfp_mask, int nid,
2836                                                 int zid)
2837 {
2838         unsigned long nr_reclaimed = 0;
2839         struct mem_cgroup_per_zone *mz, *next_mz = NULL;
2840         unsigned long reclaimed;
2841         int loop = 0;
2842         struct mem_cgroup_tree_per_zone *mctz;
2843         unsigned long long excess;
2844
2845         if (order > 0)
2846                 return 0;
2847
2848         mctz = soft_limit_tree_node_zone(nid, zid);
2849         /*
2850          * This loop can run a while, specially if mem_cgroup's continuously
2851          * keep exceeding their soft limit and putting the system under
2852          * pressure
2853          */
2854         do {
2855                 if (next_mz)
2856                         mz = next_mz;
2857                 else
2858                         mz = mem_cgroup_largest_soft_limit_node(mctz);
2859                 if (!mz)
2860                         break;
2861
2862                 reclaimed = mem_cgroup_hierarchical_reclaim(mz->mem, zone,
2863                                                 gfp_mask,
2864                                                 MEM_CGROUP_RECLAIM_SOFT);
2865                 nr_reclaimed += reclaimed;
2866                 spin_lock(&mctz->lock);
2867
2868                 /*
2869                  * If we failed to reclaim anything from this memory cgroup
2870                  * it is time to move on to the next cgroup
2871                  */
2872                 next_mz = NULL;
2873                 if (!reclaimed) {
2874                         do {
2875                                 /*
2876                                  * Loop until we find yet another one.
2877                                  *
2878                                  * By the time we get the soft_limit lock
2879                                  * again, someone might have aded the
2880                                  * group back on the RB tree. Iterate to
2881                                  * make sure we get a different mem.
2882                                  * mem_cgroup_largest_soft_limit_node returns
2883                                  * NULL if no other cgroup is present on
2884                                  * the tree
2885                                  */
2886                                 next_mz =
2887                                 __mem_cgroup_largest_soft_limit_node(mctz);
2888                                 if (next_mz == mz) {
2889                                         css_put(&next_mz->mem->css);
2890                                         next_mz = NULL;
2891                                 } else /* next_mz == NULL or other memcg */
2892                                         break;
2893                         } while (1);
2894                 }
2895                 __mem_cgroup_remove_exceeded(mz->mem, mz, mctz);
2896                 excess = res_counter_soft_limit_excess(&mz->mem->res);
2897                 /*
2898                  * One school of thought says that we should not add
2899                  * back the node to the tree if reclaim returns 0.
2900                  * But our reclaim could return 0, simply because due
2901                  * to priority we are exposing a smaller subset of
2902                  * memory to reclaim from. Consider this as a longer
2903                  * term TODO.
2904                  */
2905                 /* If excess == 0, no tree ops */
2906                 __mem_cgroup_insert_exceeded(mz->mem, mz, mctz, excess);
2907                 spin_unlock(&mctz->lock);
2908                 css_put(&mz->mem->css);
2909                 loop++;
2910                 /*
2911                  * Could not reclaim anything and there are no more
2912                  * mem cgroups to try or we seem to be looping without
2913                  * reclaiming anything.
2914                  */
2915                 if (!nr_reclaimed &&
2916                         (next_mz == NULL ||
2917                         loop > MEM_CGROUP_MAX_SOFT_LIMIT_RECLAIM_LOOPS))
2918                         break;
2919         } while (!nr_reclaimed);
2920         if (next_mz)
2921                 css_put(&next_mz->mem->css);
2922         return nr_reclaimed;
2923 }
2924
2925 /*
2926  * This routine traverse page_cgroup in given list and drop them all.
2927  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
2928  */
2929 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
2930                                 int node, int zid, enum lru_list lru)
2931 {
2932         struct zone *zone;
2933         struct mem_cgroup_per_zone *mz;
2934         struct page_cgroup *pc, *busy;
2935         unsigned long flags, loop;
2936         struct list_head *list;
2937         int ret = 0;
2938
2939         zone = &NODE_DATA(node)->node_zones[zid];
2940         mz = mem_cgroup_zoneinfo(mem, node, zid);
2941         list = &mz->lists[lru];
2942
2943         loop = MEM_CGROUP_ZSTAT(mz, lru);
2944         /* give some margin against EBUSY etc...*/
2945         loop += 256;
2946         busy = NULL;
2947         while (loop--) {
2948                 ret = 0;
2949                 spin_lock_irqsave(&zone->lru_lock, flags);
2950                 if (list_empty(list)) {
2951                         spin_unlock_irqrestore(&zone->lru_lock, flags);
2952                         break;
2953                 }
2954                 pc = list_entry(list->prev, struct page_cgroup, lru);
2955                 if (busy == pc) {
2956                         list_move(&pc->lru, list);
2957                         busy = NULL;
2958                         spin_unlock_irqrestore(&zone->lru_lock, flags);
2959                         continue;
2960                 }
2961                 spin_unlock_irqrestore(&zone->lru_lock, flags);
2962
2963                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
2964                 if (ret == -ENOMEM)
2965                         break;
2966
2967                 if (ret == -EBUSY || ret == -EINVAL) {
2968                         /* found lock contention or "pc" is obsolete. */
2969                         busy = pc;
2970                         cond_resched();
2971                 } else
2972                         busy = NULL;
2973         }
2974
2975         if (!ret && !list_empty(list))
2976                 return -EBUSY;
2977         return ret;
2978 }
2979
2980 /*
2981  * make mem_cgroup's charge to be 0 if there is no task.
2982  * This enables deleting this mem_cgroup.
2983  */
2984 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
2985 {
2986         int ret;
2987         int node, zid, shrink;
2988         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
2989         struct cgroup *cgrp = mem->css.cgroup;
2990
2991         css_get(&mem->css);
2992
2993         shrink = 0;
2994         /* should free all ? */
2995         if (free_all)
2996                 goto try_to_free;
2997 move_account:
2998         do {
2999                 ret = -EBUSY;
3000                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
3001                         goto out;
3002                 ret = -EINTR;
3003                 if (signal_pending(current))
3004                         goto out;
3005                 /* This is for making all *used* pages to be on LRU. */
3006                 lru_add_drain_all();
3007                 drain_all_stock_sync();
3008                 ret = 0;
3009                 for_each_node_state(node, N_HIGH_MEMORY) {
3010                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
3011                                 enum lru_list l;
3012                                 for_each_lru(l) {
3013                                         ret = mem_cgroup_force_empty_list(mem,
3014                                                         node, zid, l);
3015                                         if (ret)
3016                                                 break;
3017                                 }
3018                         }
3019                         if (ret)
3020                                 break;
3021                 }
3022                 memcg_oom_recover(mem);
3023                 /* it seems parent cgroup doesn't have enough mem */
3024                 if (ret == -ENOMEM)
3025                         goto try_to_free;
3026                 cond_resched();
3027         /* "ret" should also be checked to ensure all lists are empty. */
3028         } while (mem->res.usage > 0 || ret);
3029 out:
3030         css_put(&mem->css);
3031         return ret;
3032
3033 try_to_free:
3034         /* returns EBUSY if there is a task or if we come here twice. */
3035         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
3036                 ret = -EBUSY;
3037                 goto out;
3038         }
3039         /* we call try-to-free pages for make this cgroup empty */
3040         lru_add_drain_all();
3041         /* try to free all pages in this cgroup */
3042         shrink = 1;
3043         while (nr_retries && mem->res.usage > 0) {
3044                 int progress;
3045
3046                 if (signal_pending(current)) {
3047                         ret = -EINTR;
3048                         goto out;
3049                 }
3050                 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
3051                                                 false, get_swappiness(mem));
3052                 if (!progress) {
3053                         nr_retries--;
3054                         /* maybe some writeback is necessary */
3055                         congestion_wait(BLK_RW_ASYNC, HZ/10);
3056                 }
3057
3058         }
3059         lru_add_drain();
3060         /* try move_account...there may be some *locked* pages. */
3061         goto move_account;
3062 }
3063
3064 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
3065 {
3066         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
3067 }
3068
3069
3070 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
3071 {
3072         return mem_cgroup_from_cont(cont)->use_hierarchy;
3073 }
3074
3075 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
3076                                         u64 val)
3077 {
3078         int retval = 0;
3079         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3080         struct cgroup *parent = cont->parent;
3081         struct mem_cgroup *parent_mem = NULL;
3082
3083         if (parent)
3084                 parent_mem = mem_cgroup_from_cont(parent);
3085
3086         cgroup_lock();
3087         /*
3088          * If parent's use_hierarchy is set, we can't make any modifications
3089          * in the child subtrees. If it is unset, then the change can
3090          * occur, provided the current cgroup has no children.
3091          *
3092          * For the root cgroup, parent_mem is NULL, we allow value to be
3093          * set if there are no children.
3094          */
3095         if ((!parent_mem || !parent_mem->use_hierarchy) &&
3096                                 (val == 1 || val == 0)) {
3097                 if (list_empty(&cont->children))
3098                         mem->use_hierarchy = val;
3099                 else
3100                         retval = -EBUSY;
3101         } else
3102                 retval = -EINVAL;
3103         cgroup_unlock();
3104
3105         return retval;
3106 }
3107
3108 struct mem_cgroup_idx_data {
3109         s64 val;
3110         enum mem_cgroup_stat_index idx;
3111 };
3112
3113 static int
3114 mem_cgroup_get_idx_stat(struct mem_cgroup *mem, void *data)
3115 {
3116         struct mem_cgroup_idx_data *d = data;
3117         d->val += mem_cgroup_read_stat(mem, d->idx);
3118         return 0;
3119 }
3120
3121 static void
3122 mem_cgroup_get_recursive_idx_stat(struct mem_cgroup *mem,
3123                                 enum mem_cgroup_stat_index idx, s64 *val)
3124 {
3125         struct mem_cgroup_idx_data d;
3126         d.idx = idx;
3127         d.val = 0;
3128         mem_cgroup_walk_tree(mem, &d, mem_cgroup_get_idx_stat);
3129         *val = d.val;
3130 }
3131
3132 static inline u64 mem_cgroup_usage(struct mem_cgroup *mem, bool swap)
3133 {
3134         u64 idx_val, val;
3135
3136         if (!mem_cgroup_is_root(mem)) {
3137                 if (!swap)
3138                         return res_counter_read_u64(&mem->res, RES_USAGE);
3139                 else
3140                         return res_counter_read_u64(&mem->memsw, RES_USAGE);
3141         }
3142
3143         mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_CACHE, &idx_val);
3144         val = idx_val;
3145         mem_cgroup_get_recursive_idx_stat(mem, MEM_CGROUP_STAT_RSS, &idx_val);
3146         val += idx_val;
3147
3148         if (swap) {
3149                 mem_cgroup_get_recursive_idx_stat(mem,
3150                                 MEM_CGROUP_STAT_SWAPOUT, &idx_val);
3151                 val += idx_val;
3152         }
3153
3154         return val << PAGE_SHIFT;
3155 }
3156
3157 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
3158 {
3159         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
3160         u64 val;
3161         int type, name;
3162
3163         type = MEMFILE_TYPE(cft->private);
3164         name = MEMFILE_ATTR(cft->private);
3165         switch (type) {
3166         case _MEM:
3167                 if (name == RES_USAGE)
3168                         val = mem_cgroup_usage(mem, false);
3169                 else
3170                         val = res_counter_read_u64(&mem->res, name);
3171                 break;
3172         case _MEMSWAP:
3173                 if (name == RES_USAGE)
3174                         val = mem_cgroup_usage(mem, true);
3175                 else
3176                         val = res_counter_read_u64(&mem->memsw, name);
3177                 break;
3178         default:
3179                 BUG();
3180                 break;
3181         }
3182         return val;
3183 }
3184 /*
3185  * The user of this function is...
3186  * RES_LIMIT.
3187  */
3188 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
3189                             const char *buffer)
3190 {
3191         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
3192         int type, name;
3193         unsigned long long val;
3194         int ret;
3195
3196         type = MEMFILE_TYPE(cft->private);
3197         name = MEMFILE_ATTR(cft->private);
3198         switch (name) {
3199         case RES_LIMIT:
3200                 if (mem_cgroup_is_root(memcg)) { /* Can't set limit on root */
3201                         ret = -EINVAL;
3202                         break;
3203                 }
3204                 /* This function does all necessary parse...reuse it */
3205                 ret = res_counter_memparse_write_strategy(buffer, &val);
3206                 if (ret)
3207                         break;
3208                 if (type == _MEM)
3209                         ret = mem_cgroup_resize_limit(memcg, val);
3210                 else
3211                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
3212                 break;
3213         case RES_SOFT_LIMIT:
3214                 ret = res_counter_memparse_write_strategy(buffer, &val);
3215                 if (ret)
3216                         break;
3217                 /*
3218                  * For memsw, soft limits are hard to implement in terms
3219                  * of semantics, for now, we support soft limits for
3220                  * control without swap
3221                  */
3222                 if (type == _MEM)
3223                         ret = res_counter_set_soft_limit(&memcg->res, val);
3224                 else
3225                         ret = -EINVAL;
3226                 break;
3227         default:
3228                 ret = -EINVAL; /* should be BUG() ? */
3229                 break;
3230         }
3231         return ret;
3232 }
3233
3234 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
3235                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
3236 {
3237         struct cgroup *cgroup;
3238         unsigned long long min_limit, min_memsw_limit, tmp;
3239
3240         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
3241         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3242         cgroup = memcg->css.cgroup;
3243         if (!memcg->use_hierarchy)
3244                 goto out;
3245
3246         while (cgroup->parent) {
3247                 cgroup = cgroup->parent;
3248                 memcg = mem_cgroup_from_cont(cgroup);
3249                 if (!memcg->use_hierarchy)
3250                         break;
3251                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
3252                 min_limit = min(min_limit, tmp);
3253                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
3254                 min_memsw_limit = min(min_memsw_limit, tmp);
3255         }
3256 out:
3257         *mem_limit = min_limit;
3258         *memsw_limit = min_memsw_limit;
3259         return;
3260 }
3261
3262 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
3263 {
3264         struct mem_cgroup *mem;
3265         int type, name;
3266
3267         mem = mem_cgroup_from_cont(cont);
3268         type = MEMFILE_TYPE(event);
3269         name = MEMFILE_ATTR(event);
3270         switch (name) {
3271         case RES_MAX_USAGE:
3272                 if (type == _MEM)
3273                         res_counter_reset_max(&mem->res);
3274                 else
3275                         res_counter_reset_max(&mem->memsw);
3276                 break;
3277         case RES_FAILCNT:
3278                 if (type == _MEM)
3279                         res_counter_reset_failcnt(&mem->res);
3280                 else
3281                         res_counter_reset_failcnt(&mem->memsw);
3282                 break;
3283         }
3284
3285         return 0;
3286 }
3287
3288 static u64 mem_cgroup_move_charge_read(struct cgroup *cgrp,
3289                                         struct cftype *cft)
3290 {
3291         return mem_cgroup_from_cont(cgrp)->move_charge_at_immigrate;
3292 }
3293
3294 #ifdef CONFIG_MMU
3295 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3296                                         struct cftype *cft, u64 val)
3297 {
3298         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3299
3300         if (val >= (1 << NR_MOVE_TYPE))
3301                 return -EINVAL;
3302         /*
3303          * We check this value several times in both in can_attach() and
3304          * attach(), so we need cgroup lock to prevent this value from being
3305          * inconsistent.
3306          */
3307         cgroup_lock();
3308         mem->move_charge_at_immigrate = val;
3309         cgroup_unlock();
3310
3311         return 0;
3312 }
3313 #else
3314 static int mem_cgroup_move_charge_write(struct cgroup *cgrp,
3315                                         struct cftype *cft, u64 val)
3316 {
3317         return -ENOSYS;
3318 }
3319 #endif
3320
3321
3322 /* For read statistics */
3323 enum {
3324         MCS_CACHE,
3325         MCS_RSS,
3326         MCS_FILE_MAPPED,
3327         MCS_PGPGIN,
3328         MCS_PGPGOUT,
3329         MCS_SWAP,
3330         MCS_INACTIVE_ANON,
3331         MCS_ACTIVE_ANON,
3332         MCS_INACTIVE_FILE,
3333         MCS_ACTIVE_FILE,
3334         MCS_UNEVICTABLE,
3335         NR_MCS_STAT,
3336 };
3337
3338 struct mcs_total_stat {
3339         s64 stat[NR_MCS_STAT];
3340 };
3341
3342 struct {
3343         char *local_name;
3344         char *total_name;
3345 } memcg_stat_strings[NR_MCS_STAT] = {
3346         {"cache", "total_cache"},
3347         {"rss", "total_rss"},
3348         {"mapped_file", "total_mapped_file"},
3349         {"pgpgin", "total_pgpgin"},
3350         {"pgpgout", "total_pgpgout"},
3351         {"swap", "total_swap"},
3352         {"inactive_anon", "total_inactive_anon"},
3353         {"active_anon", "total_active_anon"},
3354         {"inactive_file", "total_inactive_file"},
3355         {"active_file", "total_active_file"},
3356         {"unevictable", "total_unevictable"}
3357 };
3358
3359
3360 static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
3361 {
3362         struct mcs_total_stat *s = data;
3363         s64 val;
3364
3365         /* per cpu stat */
3366         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_CACHE);
3367         s->stat[MCS_CACHE] += val * PAGE_SIZE;
3368         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_RSS);
3369         s->stat[MCS_RSS] += val * PAGE_SIZE;
3370         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_FILE_MAPPED);
3371         s->stat[MCS_FILE_MAPPED] += val * PAGE_SIZE;
3372         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGIN_COUNT);
3373         s->stat[MCS_PGPGIN] += val;
3374         val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_PGPGOUT_COUNT);
3375         s->stat[MCS_PGPGOUT] += val;
3376         if (do_swap_account) {
3377                 val = mem_cgroup_read_stat(mem, MEM_CGROUP_STAT_SWAPOUT);
3378                 s->stat[MCS_SWAP] += val * PAGE_SIZE;
3379         }
3380
3381         /* per zone stat */
3382         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
3383         s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
3384         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
3385         s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
3386         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
3387         s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
3388         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
3389         s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
3390         val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
3391         s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
3392         return 0;
3393 }
3394
3395 static void
3396 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
3397 {
3398         mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
3399 }
3400
3401 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
3402                                  struct cgroup_map_cb *cb)
3403 {
3404         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
3405         struct mcs_total_stat mystat;
3406         int i;
3407
3408         memset(&mystat, 0, sizeof(mystat));
3409         mem_cgroup_get_local_stat(mem_cont, &mystat);
3410
3411         for (i = 0; i < NR_MCS_STAT; i++) {
3412                 if (i == MCS_SWAP && !do_swap_account)
3413                         continue;
3414                 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
3415         }
3416
3417         /* Hierarchical information */
3418         {
3419                 unsigned long long limit, memsw_limit;
3420                 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
3421                 cb->fill(cb, "hierarchical_memory_limit", limit);
3422                 if (do_swap_account)
3423                         cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
3424         }
3425
3426         memset(&mystat, 0, sizeof(mystat));
3427         mem_cgroup_get_total_stat(mem_cont, &mystat);
3428         for (i = 0; i < NR_MCS_STAT; i++) {
3429                 if (i == MCS_SWAP && !do_swap_account)
3430                         continue;
3431                 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
3432         }
3433
3434 #ifdef CONFIG_DEBUG_VM
3435         cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
3436
3437         {
3438                 int nid, zid;
3439                 struct mem_cgroup_per_zone *mz;
3440                 unsigned long recent_rotated[2] = {0, 0};
3441                 unsigned long recent_scanned[2] = {0, 0};
3442
3443                 for_each_online_node(nid)
3444                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
3445                                 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
3446
3447                                 recent_rotated[0] +=
3448                                         mz->reclaim_stat.recent_rotated[0];
3449                                 recent_rotated[1] +=
3450                                         mz->reclaim_stat.recent_rotated[1];
3451                                 recent_scanned[0] +=
3452                                         mz->reclaim_stat.recent_scanned[0];
3453                                 recent_scanned[1] +=
3454                                         mz->reclaim_stat.recent_scanned[1];
3455                         }
3456                 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
3457                 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
3458                 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
3459                 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
3460         }
3461 #endif
3462
3463         return 0;
3464 }
3465
3466 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
3467 {
3468         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3469
3470         return get_swappiness(memcg);
3471 }
3472
3473 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
3474                                        u64 val)
3475 {
3476         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3477         struct mem_cgroup *parent;
3478
3479         if (val > 100)
3480                 return -EINVAL;
3481
3482         if (cgrp->parent == NULL)
3483                 return -EINVAL;
3484
3485         parent = mem_cgroup_from_cont(cgrp->parent);
3486
3487         cgroup_lock();
3488
3489         /* If under hierarchy, only empty-root can set this value */
3490         if ((parent->use_hierarchy) ||
3491             (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
3492                 cgroup_unlock();
3493                 return -EINVAL;
3494         }
3495
3496         spin_lock(&memcg->reclaim_param_lock);
3497         memcg->swappiness = val;
3498         spin_unlock(&memcg->reclaim_param_lock);
3499
3500         cgroup_unlock();
3501
3502         return 0;
3503 }
3504
3505 static void __mem_cgroup_threshold(struct mem_cgroup *memcg, bool swap)
3506 {
3507         struct mem_cgroup_threshold_ary *t;
3508         u64 usage;
3509         int i;
3510
3511         rcu_read_lock();
3512         if (!swap)
3513                 t = rcu_dereference(memcg->thresholds.primary);
3514         else
3515                 t = rcu_dereference(memcg->memsw_thresholds.primary);
3516
3517         if (!t)
3518                 goto unlock;
3519
3520         usage = mem_cgroup_usage(memcg, swap);
3521
3522         /*
3523          * current_threshold points to threshold just below usage.
3524          * If it's not true, a threshold was crossed after last
3525          * call of __mem_cgroup_threshold().
3526          */
3527         i = t->current_threshold;
3528
3529         /*
3530          * Iterate backward over array of thresholds starting from
3531          * current_threshold and check if a threshold is crossed.
3532          * If none of thresholds below usage is crossed, we read
3533          * only one element of the array here.
3534          */
3535         for (; i >= 0 && unlikely(t->entries[i].threshold > usage); i--)
3536                 eventfd_signal(t->entries[i].eventfd, 1);
3537
3538         /* i = current_threshold + 1 */
3539         i++;
3540
3541         /*
3542          * Iterate forward over array of thresholds starting from
3543          * current_threshold+1 and check if a threshold is crossed.
3544          * If none of thresholds above usage is crossed, we read
3545          * only one element of the array here.
3546          */
3547         for (; i < t->size && unlikely(t->entries[i].threshold <= usage); i++)
3548                 eventfd_signal(t->entries[i].eventfd, 1);
3549
3550         /* Update current_threshold */
3551         t->current_threshold = i - 1;
3552 unlock:
3553         rcu_read_unlock();
3554 }
3555
3556 static void mem_cgroup_threshold(struct mem_cgroup *memcg)
3557 {
3558         __mem_cgroup_threshold(memcg, false);
3559         if (do_swap_account)
3560                 __mem_cgroup_threshold(memcg, true);
3561 }
3562
3563 static int compare_thresholds(const void *a, const void *b)
3564 {
3565         const struct mem_cgroup_threshold *_a = a;
3566         const struct mem_cgroup_threshold *_b = b;
3567
3568         return _a->threshold - _b->threshold;
3569 }
3570
3571 static int mem_cgroup_oom_notify_cb(struct mem_cgroup *mem, void *data)
3572 {
3573         struct mem_cgroup_eventfd_list *ev;
3574
3575         list_for_each_entry(ev, &mem->oom_notify, list)
3576                 eventfd_signal(ev->eventfd, 1);
3577         return 0;
3578 }
3579
3580 static void mem_cgroup_oom_notify(struct mem_cgroup *mem)
3581 {
3582         mem_cgroup_walk_tree(mem, NULL, mem_cgroup_oom_notify_cb);
3583 }
3584
3585 static int mem_cgroup_usage_register_event(struct cgroup *cgrp,
3586         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3587 {
3588         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3589         struct mem_cgroup_thresholds *thresholds;
3590         struct mem_cgroup_threshold_ary *new;
3591         int type = MEMFILE_TYPE(cft->private);
3592         u64 threshold, usage;
3593         int i, size, ret;
3594
3595         ret = res_counter_memparse_write_strategy(args, &threshold);
3596         if (ret)
3597                 return ret;
3598
3599         mutex_lock(&memcg->thresholds_lock);
3600
3601         if (type == _MEM)
3602                 thresholds = &memcg->thresholds;
3603         else if (type == _MEMSWAP)
3604                 thresholds = &memcg->memsw_thresholds;
3605         else
3606                 BUG();
3607
3608         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3609
3610         /* Check if a threshold crossed before adding a new one */
3611         if (thresholds->primary)
3612                 __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3613
3614         size = thresholds->primary ? thresholds->primary->size + 1 : 1;
3615
3616         /* Allocate memory for new array of thresholds */
3617         new = kmalloc(sizeof(*new) + size * sizeof(struct mem_cgroup_threshold),
3618                         GFP_KERNEL);
3619         if (!new) {
3620                 ret = -ENOMEM;
3621                 goto unlock;
3622         }
3623         new->size = size;
3624
3625         /* Copy thresholds (if any) to new array */
3626         if (thresholds->primary) {
3627                 memcpy(new->entries, thresholds->primary->entries, (size - 1) *
3628                                 sizeof(struct mem_cgroup_threshold));
3629         }
3630
3631         /* Add new threshold */
3632         new->entries[size - 1].eventfd = eventfd;
3633         new->entries[size - 1].threshold = threshold;
3634
3635         /* Sort thresholds. Registering of new threshold isn't time-critical */
3636         sort(new->entries, size, sizeof(struct mem_cgroup_threshold),
3637                         compare_thresholds, NULL);
3638
3639         /* Find current threshold */
3640         new->current_threshold = -1;
3641         for (i = 0; i < size; i++) {
3642                 if (new->entries[i].threshold < usage) {
3643                         /*
3644                          * new->current_threshold will not be used until
3645                          * rcu_assign_pointer(), so it's safe to increment
3646                          * it here.
3647                          */
3648                         ++new->current_threshold;
3649                 }
3650         }
3651
3652         /* Free old spare buffer and save old primary buffer as spare */
3653         kfree(thresholds->spare);
3654         thresholds->spare = thresholds->primary;
3655
3656         rcu_assign_pointer(thresholds->primary, new);
3657
3658         /* To be sure that nobody uses thresholds */
3659         synchronize_rcu();
3660
3661 unlock:
3662         mutex_unlock(&memcg->thresholds_lock);
3663
3664         return ret;
3665 }
3666
3667 static void mem_cgroup_usage_unregister_event(struct cgroup *cgrp,
3668         struct cftype *cft, struct eventfd_ctx *eventfd)
3669 {
3670         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3671         struct mem_cgroup_thresholds *thresholds;
3672         struct mem_cgroup_threshold_ary *new;
3673         int type = MEMFILE_TYPE(cft->private);
3674         u64 usage;
3675         int i, j, size;
3676
3677         mutex_lock(&memcg->thresholds_lock);
3678         if (type == _MEM)
3679                 thresholds = &memcg->thresholds;
3680         else if (type == _MEMSWAP)
3681                 thresholds = &memcg->memsw_thresholds;
3682         else
3683                 BUG();
3684
3685         /*
3686          * Something went wrong if we trying to unregister a threshold
3687          * if we don't have thresholds
3688          */
3689         BUG_ON(!thresholds);
3690
3691         usage = mem_cgroup_usage(memcg, type == _MEMSWAP);
3692
3693         /* Check if a threshold crossed before removing */
3694         __mem_cgroup_threshold(memcg, type == _MEMSWAP);
3695
3696         /* Calculate new number of threshold */
3697         size = 0;
3698         for (i = 0; i < thresholds->primary->size; i++) {
3699                 if (thresholds->primary->entries[i].eventfd != eventfd)
3700                         size++;
3701         }
3702
3703         new = thresholds->spare;
3704
3705         /* Set thresholds array to NULL if we don't have thresholds */
3706         if (!size) {
3707                 kfree(new);
3708                 new = NULL;
3709                 goto swap_buffers;
3710         }
3711
3712         new->size = size;
3713
3714         /* Copy thresholds and find current threshold */
3715         new->current_threshold = -1;
3716         for (i = 0, j = 0; i < thresholds->primary->size; i++) {
3717                 if (thresholds->primary->entries[i].eventfd == eventfd)
3718                         continue;
3719
3720                 new->entries[j] = thresholds->primary->entries[i];
3721                 if (new->entries[j].threshold < usage) {
3722                         /*
3723                          * new->current_threshold will not be used
3724                          * until rcu_assign_pointer(), so it's safe to increment
3725                          * it here.
3726                          */
3727                         ++new->current_threshold;
3728                 }
3729                 j++;
3730         }
3731
3732 swap_buffers:
3733         /* Swap primary and spare array */
3734         thresholds->spare = thresholds->primary;
3735         rcu_assign_pointer(thresholds->primary, new);
3736
3737         /* To be sure that nobody uses thresholds */
3738         synchronize_rcu();
3739
3740         mutex_unlock(&memcg->thresholds_lock);
3741 }
3742
3743 static int mem_cgroup_oom_register_event(struct cgroup *cgrp,
3744         struct cftype *cft, struct eventfd_ctx *eventfd, const char *args)
3745 {
3746         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
3747         struct mem_cgroup_eventfd_list *event;
3748         int type = MEMFILE_TYPE(cft->private);
3749
3750         BUG_ON(type != _OOM_TYPE);
3751         event = kmalloc(sizeof(*event), GFP_KERNEL);
3752         if (!event)
3753                 return -ENOMEM;
3754
3755         mutex_lock(&memcg_oom_mutex);
3756
3757         event->eventfd = eventfd;
3758         list_add(&event->list, &memcg->oom_notify);
3759
3760         /* already in OOM ? */
3761         if (atomic_read(&memcg->oom_lock))
3762                 eventfd_signal(eventfd, 1);
3763         mutex_unlock(&memcg_oom_mutex);
3764
3765         return 0;
3766 }
3767
3768 static void mem_cgroup_oom_unregister_event(struct cgroup *cgrp,
3769         struct cftype *cft, struct eventfd_ctx *eventfd)
3770 {
3771         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3772         struct mem_cgroup_eventfd_list *ev, *tmp;
3773         int type = MEMFILE_TYPE(cft->private);
3774
3775         BUG_ON(type != _OOM_TYPE);
3776
3777         mutex_lock(&memcg_oom_mutex);
3778
3779         list_for_each_entry_safe(ev, tmp, &mem->oom_notify, list) {
3780                 if (ev->eventfd == eventfd) {
3781                         list_del(&ev->list);
3782                         kfree(ev);
3783                 }
3784         }
3785
3786         mutex_unlock(&memcg_oom_mutex);
3787 }
3788
3789 static int mem_cgroup_oom_control_read(struct cgroup *cgrp,
3790         struct cftype *cft,  struct cgroup_map_cb *cb)
3791 {
3792         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3793
3794         cb->fill(cb, "oom_kill_disable", mem->oom_kill_disable);
3795
3796         if (atomic_read(&mem->oom_lock))
3797                 cb->fill(cb, "under_oom", 1);
3798         else
3799                 cb->fill(cb, "under_oom", 0);
3800         return 0;
3801 }
3802
3803 /*
3804  */
3805 static int mem_cgroup_oom_control_write(struct cgroup *cgrp,
3806         struct cftype *cft, u64 val)
3807 {
3808         struct mem_cgroup *mem = mem_cgroup_from_cont(cgrp);
3809         struct mem_cgroup *parent;
3810
3811         /* cannot set to root cgroup and only 0 and 1 are allowed */
3812         if (!cgrp->parent || !((val == 0) || (val == 1)))
3813                 return -EINVAL;
3814
3815         parent = mem_cgroup_from_cont(cgrp->parent);
3816
3817         cgroup_lock();
3818         /* oom-kill-disable is a flag for subhierarchy. */
3819         if ((parent->use_hierarchy) ||
3820             (mem->use_hierarchy && !list_empty(&cgrp->children))) {
3821                 cgroup_unlock();
3822                 return -EINVAL;
3823         }
3824         mem->oom_kill_disable = val;
3825         if (!val)
3826                 memcg_oom_recover(mem);
3827         cgroup_unlock();
3828         return 0;
3829 }
3830
3831 static struct cftype mem_cgroup_files[] = {
3832         {
3833                 .name = "usage_in_bytes",
3834                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
3835                 .read_u64 = mem_cgroup_read,
3836                 .register_event = mem_cgroup_usage_register_event,
3837                 .unregister_event = mem_cgroup_usage_unregister_event,
3838         },
3839         {
3840                 .name = "max_usage_in_bytes",
3841                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
3842                 .trigger = mem_cgroup_reset,
3843                 .read_u64 = mem_cgroup_read,
3844         },
3845         {
3846                 .name = "limit_in_bytes",
3847                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
3848                 .write_string = mem_cgroup_write,
3849                 .read_u64 = mem_cgroup_read,
3850         },
3851         {
3852                 .name = "soft_limit_in_bytes",
3853                 .private = MEMFILE_PRIVATE(_MEM, RES_SOFT_LIMIT),
3854                 .write_string = mem_cgroup_write,
3855                 .read_u64 = mem_cgroup_read,
3856         },
3857         {
3858                 .name = "failcnt",
3859                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
3860                 .trigger = mem_cgroup_reset,
3861                 .read_u64 = mem_cgroup_read,
3862         },
3863         {
3864                 .name = "stat",
3865                 .read_map = mem_control_stat_show,
3866         },
3867         {
3868                 .name = "force_empty",
3869                 .trigger = mem_cgroup_force_empty_write,
3870         },
3871         {
3872                 .name = "use_hierarchy",
3873                 .write_u64 = mem_cgroup_hierarchy_write,
3874                 .read_u64 = mem_cgroup_hierarchy_read,
3875         },
3876         {
3877                 .name = "swappiness",
3878                 .read_u64 = mem_cgroup_swappiness_read,
3879                 .write_u64 = mem_cgroup_swappiness_write,
3880         },
3881         {
3882                 .name = "move_charge_at_immigrate",
3883                 .read_u64 = mem_cgroup_move_charge_read,
3884                 .write_u64 = mem_cgroup_move_charge_write,
3885         },
3886         {
3887                 .name = "oom_control",
3888                 .read_map = mem_cgroup_oom_control_read,
3889                 .write_u64 = mem_cgroup_oom_control_write,
3890                 .register_event = mem_cgroup_oom_register_event,
3891                 .unregister_event = mem_cgroup_oom_unregister_event,
3892                 .private = MEMFILE_PRIVATE(_OOM_TYPE, OOM_CONTROL),
3893         },
3894 };
3895
3896 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
3897 static struct cftype memsw_cgroup_files[] = {
3898         {
3899                 .name = "memsw.usage_in_bytes",
3900                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
3901                 .read_u64 = mem_cgroup_read,
3902                 .register_event = mem_cgroup_usage_register_event,
3903                 .unregister_event = mem_cgroup_usage_unregister_event,
3904         },
3905         {
3906                 .name = "memsw.max_usage_in_bytes",
3907                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
3908                 .trigger = mem_cgroup_reset,
3909                 .read_u64 = mem_cgroup_read,
3910         },
3911         {
3912                 .name = "memsw.limit_in_bytes",
3913                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
3914                 .write_string = mem_cgroup_write,
3915                 .read_u64 = mem_cgroup_read,
3916         },
3917         {
3918                 .name = "memsw.failcnt",
3919                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
3920                 .trigger = mem_cgroup_reset,
3921                 .read_u64 = mem_cgroup_read,
3922         },
3923 };
3924
3925 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
3926 {
3927         if (!do_swap_account)
3928                 return 0;
3929         return cgroup_add_files(cont, ss, memsw_cgroup_files,
3930                                 ARRAY_SIZE(memsw_cgroup_files));
3931 };
3932 #else
3933 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
3934 {
3935         return 0;
3936 }
3937 #endif
3938
3939 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
3940 {
3941         struct mem_cgroup_per_node *pn;
3942         struct mem_cgroup_per_zone *mz;
3943         enum lru_list l;
3944         int zone, tmp = node;
3945         /*
3946          * This routine is called against possible nodes.
3947          * But it's BUG to call kmalloc() against offline node.
3948          *
3949          * TODO: this routine can waste much memory for nodes which will
3950          *       never be onlined. It's better to use memory hotplug callback
3951          *       function.
3952          */
3953         if (!node_state(node, N_NORMAL_MEMORY))
3954                 tmp = -1;
3955         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
3956         if (!pn)
3957                 return 1;
3958
3959         mem->info.nodeinfo[node] = pn;
3960         memset(pn, 0, sizeof(*pn));
3961
3962         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
3963                 mz = &pn->zoneinfo[zone];
3964                 for_each_lru(l)
3965                         INIT_LIST_HEAD(&mz->lists[l]);
3966                 mz->usage_in_excess = 0;
3967                 mz->on_tree = false;
3968                 mz->mem = mem;
3969         }
3970         return 0;
3971 }
3972
3973 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
3974 {
3975         kfree(mem->info.nodeinfo[node]);
3976 }
3977
3978 static struct mem_cgroup *mem_cgroup_alloc(void)
3979 {
3980         struct mem_cgroup *mem;
3981         int size = sizeof(struct mem_cgroup);
3982
3983         /* Can be very big if MAX_NUMNODES is very big */
3984         if (size < PAGE_SIZE)
3985                 mem = kmalloc(size, GFP_KERNEL);
3986         else
3987                 mem = vmalloc(size);
3988
3989         if (!mem)
3990                 return NULL;
3991
3992         memset(mem, 0, size);
3993         mem->stat = alloc_percpu(struct mem_cgroup_stat_cpu);
3994         if (!mem->stat) {
3995                 if (size < PAGE_SIZE)
3996                         kfree(mem);
3997                 else
3998                         vfree(mem);
3999                 mem = NULL;
4000         }
4001         return mem;
4002 }
4003
4004 /*
4005  * At destroying mem_cgroup, references from swap_cgroup can remain.
4006  * (scanning all at force_empty is too costly...)
4007  *
4008  * Instead of clearing all references at force_empty, we remember
4009  * the number of reference from swap_cgroup and free mem_cgroup when
4010  * it goes down to 0.
4011  *
4012  * Removal of cgroup itself succeeds regardless of refs from swap.
4013  */
4014
4015 static void __mem_cgroup_free(struct mem_cgroup *mem)
4016 {
4017         int node;
4018
4019         mem_cgroup_remove_from_trees(mem);
4020         free_css_id(&mem_cgroup_subsys, &mem->css);
4021
4022         for_each_node_state(node, N_POSSIBLE)
4023                 free_mem_cgroup_per_zone_info(mem, node);
4024
4025         free_percpu(mem->stat);
4026         if (sizeof(struct mem_cgroup) < PAGE_SIZE)
4027                 kfree(mem);
4028         else
4029                 vfree(mem);
4030 }
4031
4032 static void mem_cgroup_get(struct mem_cgroup *mem)
4033 {
4034         atomic_inc(&mem->refcnt);
4035 }
4036
4037 static void __mem_cgroup_put(struct mem_cgroup *mem, int count)
4038 {
4039         if (atomic_sub_and_test(count, &mem->refcnt)) {
4040                 struct mem_cgroup *parent = parent_mem_cgroup(mem);
4041                 __mem_cgroup_free(mem);
4042                 if (parent)
4043                         mem_cgroup_put(parent);
4044         }
4045 }
4046
4047 static void mem_cgroup_put(struct mem_cgroup *mem)
4048 {
4049         __mem_cgroup_put(mem, 1);
4050 }
4051
4052 /*
4053  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
4054  */
4055 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
4056 {
4057         if (!mem->res.parent)
4058                 return NULL;
4059         return mem_cgroup_from_res_counter(mem->res.parent, res);
4060 }
4061
4062 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4063 static void __init enable_swap_cgroup(void)
4064 {
4065         if (!mem_cgroup_disabled() && really_do_swap_account)
4066                 do_swap_account = 1;
4067 }
4068 #else
4069 static void __init enable_swap_cgroup(void)
4070 {
4071 }
4072 #endif
4073
4074 static int mem_cgroup_soft_limit_tree_init(void)
4075 {
4076         struct mem_cgroup_tree_per_node *rtpn;
4077         struct mem_cgroup_tree_per_zone *rtpz;
4078         int tmp, node, zone;
4079
4080         for_each_node_state(node, N_POSSIBLE) {
4081                 tmp = node;
4082                 if (!node_state(node, N_NORMAL_MEMORY))
4083                         tmp = -1;
4084                 rtpn = kzalloc_node(sizeof(*rtpn), GFP_KERNEL, tmp);
4085                 if (!rtpn)
4086                         return 1;
4087
4088                 soft_limit_tree.rb_tree_per_node[node] = rtpn;
4089
4090                 for (zone = 0; zone < MAX_NR_ZONES; zone++) {
4091                         rtpz = &rtpn->rb_tree_per_zone[zone];
4092                         rtpz->rb_root = RB_ROOT;
4093                         spin_lock_init(&rtpz->lock);
4094                 }
4095         }
4096         return 0;
4097 }
4098
4099 static struct cgroup_subsys_state * __ref
4100 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
4101 {
4102         struct mem_cgroup *mem, *parent;
4103         long error = -ENOMEM;
4104         int node;
4105
4106         mem = mem_cgroup_alloc();
4107         if (!mem)
4108                 return ERR_PTR(error);
4109
4110         for_each_node_state(node, N_POSSIBLE)
4111                 if (alloc_mem_cgroup_per_zone_info(mem, node))
4112                         goto free_out;
4113
4114         /* root ? */
4115         if (cont->parent == NULL) {
4116                 int cpu;
4117                 enable_swap_cgroup();
4118                 parent = NULL;
4119                 root_mem_cgroup = mem;
4120                 if (mem_cgroup_soft_limit_tree_init())
4121                         goto free_out;
4122                 for_each_possible_cpu(cpu) {
4123                         struct memcg_stock_pcp *stock =
4124                                                 &per_cpu(memcg_stock, cpu);
4125                         INIT_WORK(&stock->work, drain_local_stock);
4126                 }
4127                 hotcpu_notifier(memcg_stock_cpu_callback, 0);
4128         } else {
4129                 parent = mem_cgroup_from_cont(cont->parent);
4130                 mem->use_hierarchy = parent->use_hierarchy;
4131                 mem->oom_kill_disable = parent->oom_kill_disable;
4132         }
4133
4134         if (parent && parent->use_hierarchy) {
4135                 res_counter_init(&mem->res, &parent->res);
4136                 res_counter_init(&mem->memsw, &parent->memsw);
4137                 /*
4138                  * We increment refcnt of the parent to ensure that we can
4139                  * safely access it on res_counter_charge/uncharge.
4140                  * This refcnt will be decremented when freeing this
4141                  * mem_cgroup(see mem_cgroup_put).
4142                  */
4143                 mem_cgroup_get(parent);
4144         } else {
4145                 res_counter_init(&mem->res, NULL);
4146                 res_counter_init(&mem->memsw, NULL);
4147         }
4148         mem->last_scanned_child = 0;
4149         spin_lock_init(&mem->reclaim_param_lock);
4150         INIT_LIST_HEAD(&mem->oom_notify);
4151
4152         if (parent)
4153                 mem->swappiness = get_swappiness(parent);
4154         atomic_set(&mem->refcnt, 1);
4155         mem->move_charge_at_immigrate = 0;
4156         mutex_init(&mem->thresholds_lock);
4157         return &mem->css;
4158 free_out:
4159         __mem_cgroup_free(mem);
4160         root_mem_cgroup = NULL;
4161         return ERR_PTR(error);
4162 }
4163
4164 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
4165                                         struct cgroup *cont)
4166 {
4167         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4168
4169         return mem_cgroup_force_empty(mem, false);
4170 }
4171
4172 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
4173                                 struct cgroup *cont)
4174 {
4175         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
4176
4177         mem_cgroup_put(mem);
4178 }
4179
4180 static int mem_cgroup_populate(struct cgroup_subsys *ss,
4181                                 struct cgroup *cont)
4182 {
4183         int ret;
4184
4185         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
4186                                 ARRAY_SIZE(mem_cgroup_files));
4187
4188         if (!ret)
4189                 ret = register_memsw_files(cont, ss);
4190         return ret;
4191 }
4192
4193 #ifdef CONFIG_MMU
4194 /* Handlers for move charge at task migration. */
4195 #define PRECHARGE_COUNT_AT_ONCE 256
4196 static int mem_cgroup_do_precharge(unsigned long count)
4197 {
4198         int ret = 0;
4199         int batch_count = PRECHARGE_COUNT_AT_ONCE;
4200         struct mem_cgroup *mem = mc.to;
4201
4202         if (mem_cgroup_is_root(mem)) {
4203                 mc.precharge += count;
4204                 /* we don't need css_get for root */
4205                 return ret;
4206         }
4207         /* try to charge at once */
4208         if (count > 1) {
4209                 struct res_counter *dummy;
4210                 /*
4211                  * "mem" cannot be under rmdir() because we've already checked
4212                  * by cgroup_lock_live_cgroup() that it is not removed and we
4213                  * are still under the same cgroup_mutex. So we can postpone
4214                  * css_get().
4215                  */
4216                 if (res_counter_charge(&mem->res, PAGE_SIZE * count, &dummy))
4217                         goto one_by_one;
4218                 if (do_swap_account && res_counter_charge(&mem->memsw,
4219                                                 PAGE_SIZE * count, &dummy)) {
4220                         res_counter_uncharge(&mem->res, PAGE_SIZE * count);
4221                         goto one_by_one;
4222                 }
4223                 mc.precharge += count;
4224                 VM_BUG_ON(test_bit(CSS_ROOT, &mem->css.flags));
4225                 WARN_ON_ONCE(count > INT_MAX);
4226                 __css_get(&mem->css, (int)count);
4227                 return ret;
4228         }
4229 one_by_one:
4230         /* fall back to one by one charge */
4231         while (count--) {
4232                 if (signal_pending(current)) {
4233                         ret = -EINTR;
4234                         break;
4235                 }
4236                 if (!batch_count--) {
4237                         batch_count = PRECHARGE_COUNT_AT_ONCE;
4238                         cond_resched();
4239                 }
4240                 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
4241                 if (ret || !mem)
4242                         /* mem_cgroup_clear_mc() will do uncharge later */
4243                         return -ENOMEM;
4244                 mc.precharge++;
4245         }
4246         return ret;
4247 }
4248
4249 /**
4250  * is_target_pte_for_mc - check a pte whether it is valid for move charge
4251  * @vma: the vma the pte to be checked belongs
4252  * @addr: the address corresponding to the pte to be checked
4253  * @ptent: the pte to be checked
4254  * @target: the pointer the target page or swap ent will be stored(can be NULL)
4255  *
4256  * Returns
4257  *   0(MC_TARGET_NONE): if the pte is not a target for move charge.
4258  *   1(MC_TARGET_PAGE): if the page corresponding to this pte is a target for
4259  *     move charge. if @target is not NULL, the page is stored in target->page
4260  *     with extra refcnt got(Callers should handle it).
4261  *   2(MC_TARGET_SWAP): if the swap entry corresponding to this pte is a
4262  *     target for charge migration. if @target is not NULL, the entry is stored
4263  *     in target->ent.
4264  *
4265  * Called with pte lock held.
4266  */
4267 union mc_target {
4268         struct page     *page;
4269         swp_entry_t     ent;
4270 };
4271
4272 enum mc_target_type {
4273         MC_TARGET_NONE, /* not used */
4274         MC_TARGET_PAGE,
4275         MC_TARGET_SWAP,
4276 };
4277
4278 static struct page *mc_handle_present_pte(struct vm_area_struct *vma,
4279                                                 unsigned long addr, pte_t ptent)
4280 {
4281         struct page *page = vm_normal_page(vma, addr, ptent);
4282
4283         if (!page || !page_mapped(page))
4284                 return NULL;
4285         if (PageAnon(page)) {
4286                 /* we don't move shared anon */
4287                 if (!move_anon() || page_mapcount(page) > 2)
4288                         return NULL;
4289         } else if (!move_file())
4290                 /* we ignore mapcount for file pages */
4291                 return NULL;
4292         if (!get_page_unless_zero(page))
4293                 return NULL;
4294
4295         return page;
4296 }
4297
4298 static struct page *mc_handle_swap_pte(struct vm_area_struct *vma,
4299                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4300 {
4301         int usage_count;
4302         struct page *page = NULL;
4303         swp_entry_t ent = pte_to_swp_entry(ptent);
4304
4305         if (!move_anon() || non_swap_entry(ent))
4306                 return NULL;
4307         usage_count = mem_cgroup_count_swap_user(ent, &page);
4308         if (usage_count > 1) { /* we don't move shared anon */
4309                 if (page)
4310                         put_page(page);
4311                 return NULL;
4312         }
4313         if (do_swap_account)
4314                 entry->val = ent.val;
4315
4316         return page;
4317 }
4318
4319 static struct page *mc_handle_file_pte(struct vm_area_struct *vma,
4320                         unsigned long addr, pte_t ptent, swp_entry_t *entry)
4321 {
4322         struct page *page = NULL;
4323         struct inode *inode;
4324         struct address_space *mapping;
4325         pgoff_t pgoff;
4326
4327         if (!vma->vm_file) /* anonymous vma */
4328                 return NULL;
4329         if (!move_file())
4330                 return NULL;
4331
4332         inode = vma->vm_file->f_path.dentry->d_inode;
4333         mapping = vma->vm_file->f_mapping;
4334         if (pte_none(ptent))
4335                 pgoff = linear_page_index(vma, addr);
4336         else /* pte_file(ptent) is true */
4337                 pgoff = pte_to_pgoff(ptent);
4338
4339         /* page is moved even if it's not RSS of this task(page-faulted). */
4340         if (!mapping_cap_swap_backed(mapping)) { /* normal file */
4341                 page = find_get_page(mapping, pgoff);
4342         } else { /* shmem/tmpfs file. we should take account of swap too. */
4343                 swp_entry_t ent;
4344                 mem_cgroup_get_shmem_target(inode, pgoff, &page, &ent);
4345                 if (do_swap_account)
4346                         entry->val = ent.val;
4347         }
4348
4349         return page;
4350 }
4351
4352 static int is_target_pte_for_mc(struct vm_area_struct *vma,
4353                 unsigned long addr, pte_t ptent, union mc_target *target)
4354 {
4355         struct page *page = NULL;
4356         struct page_cgroup *pc;
4357         int ret = 0;
4358         swp_entry_t ent = { .val = 0 };
4359
4360         if (pte_present(ptent))
4361                 page = mc_handle_present_pte(vma, addr, ptent);
4362         else if (is_swap_pte(ptent))
4363                 page = mc_handle_swap_pte(vma, addr, ptent, &ent);
4364         else if (pte_none(ptent) || pte_file(ptent))
4365                 page = mc_handle_file_pte(vma, addr, ptent, &ent);
4366
4367         if (!page && !ent.val)
4368                 return 0;
4369         if (page) {
4370                 pc = lookup_page_cgroup(page);
4371                 /*
4372                  * Do only loose check w/o page_cgroup lock.
4373                  * mem_cgroup_move_account() checks the pc is valid or not under
4374                  * the lock.
4375                  */
4376                 if (PageCgroupUsed(pc) && pc->mem_cgroup == mc.from) {
4377                         ret = MC_TARGET_PAGE;
4378                         if (target)
4379                                 target->page = page;
4380                 }
4381                 if (!ret || !target)
4382                         put_page(page);
4383         }
4384         /* There is a swap entry and a page doesn't exist or isn't charged */
4385         if (ent.val && !ret &&
4386                         css_id(&mc.from->css) == lookup_swap_cgroup(ent)) {
4387                 ret = MC_TARGET_SWAP;
4388                 if (target)
4389                         target->ent = ent;
4390         }
4391         return ret;
4392 }
4393
4394 static int mem_cgroup_count_precharge_pte_range(pmd_t *pmd,
4395                                         unsigned long addr, unsigned long end,
4396                                         struct mm_walk *walk)
4397 {
4398         struct vm_area_struct *vma = walk->private;
4399         pte_t *pte;
4400         spinlock_t *ptl;
4401
4402         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4403         for (; addr != end; pte++, addr += PAGE_SIZE)
4404                 if (is_target_pte_for_mc(vma, addr, *pte, NULL))
4405                         mc.precharge++; /* increment precharge temporarily */
4406         pte_unmap_unlock(pte - 1, ptl);
4407         cond_resched();
4408
4409         return 0;
4410 }
4411
4412 static unsigned long mem_cgroup_count_precharge(struct mm_struct *mm)
4413 {
4414         unsigned long precharge;
4415         struct vm_area_struct *vma;
4416
4417         down_read(&mm->mmap_sem);
4418         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4419                 struct mm_walk mem_cgroup_count_precharge_walk = {
4420                         .pmd_entry = mem_cgroup_count_precharge_pte_range,
4421                         .mm = mm,
4422                         .private = vma,
4423                 };
4424                 if (is_vm_hugetlb_page(vma))
4425                         continue;
4426                 walk_page_range(vma->vm_start, vma->vm_end,
4427                                         &mem_cgroup_count_precharge_walk);
4428         }
4429         up_read(&mm->mmap_sem);
4430
4431         precharge = mc.precharge;
4432         mc.precharge = 0;
4433
4434         return precharge;
4435 }
4436
4437 static int mem_cgroup_precharge_mc(struct mm_struct *mm)
4438 {
4439         return mem_cgroup_do_precharge(mem_cgroup_count_precharge(mm));
4440 }
4441
4442 static void mem_cgroup_clear_mc(void)
4443 {
4444         /* we must uncharge all the leftover precharges from mc.to */
4445         if (mc.precharge) {
4446                 __mem_cgroup_cancel_charge(mc.to, mc.precharge);
4447                 mc.precharge = 0;
4448                 memcg_oom_recover(mc.to);
4449         }
4450         /*
4451          * we didn't uncharge from mc.from at mem_cgroup_move_account(), so
4452          * we must uncharge here.
4453          */
4454         if (mc.moved_charge) {
4455                 __mem_cgroup_cancel_charge(mc.from, mc.moved_charge);
4456                 mc.moved_charge = 0;
4457                 memcg_oom_recover(mc.from);
4458         }
4459         /* we must fixup refcnts and charges */
4460         if (mc.moved_swap) {
4461                 WARN_ON_ONCE(mc.moved_swap > INT_MAX);
4462                 /* uncharge swap account from the old cgroup */
4463                 if (!mem_cgroup_is_root(mc.from))
4464                         res_counter_uncharge(&mc.from->memsw,
4465                                                 PAGE_SIZE * mc.moved_swap);
4466                 __mem_cgroup_put(mc.from, mc.moved_swap);
4467
4468                 if (!mem_cgroup_is_root(mc.to)) {
4469                         /*
4470                          * we charged both to->res and to->memsw, so we should
4471                          * uncharge to->res.
4472                          */
4473                         res_counter_uncharge(&mc.to->res,
4474                                                 PAGE_SIZE * mc.moved_swap);
4475                         VM_BUG_ON(test_bit(CSS_ROOT, &mc.to->css.flags));
4476                         __css_put(&mc.to->css, mc.moved_swap);
4477                 }
4478                 /* we've already done mem_cgroup_get(mc.to) */
4479
4480                 mc.moved_swap = 0;
4481         }
4482         mc.from = NULL;
4483         mc.to = NULL;
4484         mc.moving_task = NULL;
4485         wake_up_all(&mc.waitq);
4486 }
4487
4488 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4489                                 struct cgroup *cgroup,
4490                                 struct task_struct *p,
4491                                 bool threadgroup)
4492 {
4493         int ret = 0;
4494         struct mem_cgroup *mem = mem_cgroup_from_cont(cgroup);
4495
4496         if (mem->move_charge_at_immigrate) {
4497                 struct mm_struct *mm;
4498                 struct mem_cgroup *from = mem_cgroup_from_task(p);
4499
4500                 VM_BUG_ON(from == mem);
4501
4502                 mm = get_task_mm(p);
4503                 if (!mm)
4504                         return 0;
4505                 /* We move charges only when we move a owner of the mm */
4506                 if (mm->owner == p) {
4507                         VM_BUG_ON(mc.from);
4508                         VM_BUG_ON(mc.to);
4509                         VM_BUG_ON(mc.precharge);
4510                         VM_BUG_ON(mc.moved_charge);
4511                         VM_BUG_ON(mc.moved_swap);
4512                         VM_BUG_ON(mc.moving_task);
4513                         mc.from = from;
4514                         mc.to = mem;
4515                         mc.precharge = 0;
4516                         mc.moved_charge = 0;
4517                         mc.moved_swap = 0;
4518                         mc.moving_task = current;
4519
4520                         ret = mem_cgroup_precharge_mc(mm);
4521                         if (ret)
4522                                 mem_cgroup_clear_mc();
4523                 }
4524                 mmput(mm);
4525         }
4526         return ret;
4527 }
4528
4529 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4530                                 struct cgroup *cgroup,
4531                                 struct task_struct *p,
4532                                 bool threadgroup)
4533 {
4534         mem_cgroup_clear_mc();
4535 }
4536
4537 static int mem_cgroup_move_charge_pte_range(pmd_t *pmd,
4538                                 unsigned long addr, unsigned long end,
4539                                 struct mm_walk *walk)
4540 {
4541         int ret = 0;
4542         struct vm_area_struct *vma = walk->private;
4543         pte_t *pte;
4544         spinlock_t *ptl;
4545
4546 retry:
4547         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
4548         for (; addr != end; addr += PAGE_SIZE) {
4549                 pte_t ptent = *(pte++);
4550                 union mc_target target;
4551                 int type;
4552                 struct page *page;
4553                 struct page_cgroup *pc;
4554                 swp_entry_t ent;
4555
4556                 if (!mc.precharge)
4557                         break;
4558
4559                 type = is_target_pte_for_mc(vma, addr, ptent, &target);
4560                 switch (type) {
4561                 case MC_TARGET_PAGE:
4562                         page = target.page;
4563                         if (isolate_lru_page(page))
4564                                 goto put;
4565                         pc = lookup_page_cgroup(page);
4566                         if (!mem_cgroup_move_account(pc,
4567                                                 mc.from, mc.to, false)) {
4568                                 mc.precharge--;
4569                                 /* we uncharge from mc.from later. */
4570                                 mc.moved_charge++;
4571                         }
4572                         putback_lru_page(page);
4573 put:                    /* is_target_pte_for_mc() gets the page */
4574                         put_page(page);
4575                         break;
4576                 case MC_TARGET_SWAP:
4577                         ent = target.ent;
4578                         if (!mem_cgroup_move_swap_account(ent,
4579                                                 mc.from, mc.to, false)) {
4580                                 mc.precharge--;
4581                                 /* we fixup refcnts and charges later. */
4582                                 mc.moved_swap++;
4583                         }
4584                         break;
4585                 default:
4586                         break;
4587                 }
4588         }
4589         pte_unmap_unlock(pte - 1, ptl);
4590         cond_resched();
4591
4592         if (addr != end) {
4593                 /*
4594                  * We have consumed all precharges we got in can_attach().
4595                  * We try charge one by one, but don't do any additional
4596                  * charges to mc.to if we have failed in charge once in attach()
4597                  * phase.
4598                  */
4599                 ret = mem_cgroup_do_precharge(1);
4600                 if (!ret)
4601                         goto retry;
4602         }
4603
4604         return ret;
4605 }
4606
4607 static void mem_cgroup_move_charge(struct mm_struct *mm)
4608 {
4609         struct vm_area_struct *vma;
4610
4611         lru_add_drain_all();
4612         down_read(&mm->mmap_sem);
4613         for (vma = mm->mmap; vma; vma = vma->vm_next) {
4614                 int ret;
4615                 struct mm_walk mem_cgroup_move_charge_walk = {
4616                         .pmd_entry = mem_cgroup_move_charge_pte_range,
4617                         .mm = mm,
4618                         .private = vma,
4619                 };
4620                 if (is_vm_hugetlb_page(vma))
4621                         continue;
4622                 ret = walk_page_range(vma->vm_start, vma->vm_end,
4623                                                 &mem_cgroup_move_charge_walk);
4624                 if (ret)
4625                         /*
4626                          * means we have consumed all precharges and failed in
4627                          * doing additional charge. Just abandon here.
4628                          */
4629                         break;
4630         }
4631         up_read(&mm->mmap_sem);
4632 }
4633
4634 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4635                                 struct cgroup *cont,
4636                                 struct cgroup *old_cont,
4637                                 struct task_struct *p,
4638                                 bool threadgroup)
4639 {
4640         struct mm_struct *mm;
4641
4642         if (!mc.to)
4643                 /* no need to move charge */
4644                 return;
4645
4646         mm = get_task_mm(p);
4647         if (mm) {
4648                 mem_cgroup_move_charge(mm);
4649                 mmput(mm);
4650         }
4651         mem_cgroup_clear_mc();
4652 }
4653 #else   /* !CONFIG_MMU */
4654 static int mem_cgroup_can_attach(struct cgroup_subsys *ss,
4655                                 struct cgroup *cgroup,
4656                                 struct task_struct *p,
4657                                 bool threadgroup)
4658 {
4659         return 0;
4660 }
4661 static void mem_cgroup_cancel_attach(struct cgroup_subsys *ss,
4662                                 struct cgroup *cgroup,
4663                                 struct task_struct *p,
4664                                 bool threadgroup)
4665 {
4666 }
4667 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
4668                                 struct cgroup *cont,
4669                                 struct cgroup *old_cont,
4670                                 struct task_struct *p,
4671                                 bool threadgroup)
4672 {
4673 }
4674 #endif
4675
4676 struct cgroup_subsys mem_cgroup_subsys = {
4677         .name = "memory",
4678         .subsys_id = mem_cgroup_subsys_id,
4679         .create = mem_cgroup_create,
4680         .pre_destroy = mem_cgroup_pre_destroy,
4681         .destroy = mem_cgroup_destroy,
4682         .populate = mem_cgroup_populate,
4683         .can_attach = mem_cgroup_can_attach,
4684         .cancel_attach = mem_cgroup_cancel_attach,
4685         .attach = mem_cgroup_move_task,
4686         .early_init = 0,
4687         .use_id = 1,
4688 };
4689
4690 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
4691
4692 static int __init disable_swap_account(char *s)
4693 {
4694         really_do_swap_account = 0;
4695         return 1;
4696 }
4697 __setup("noswapaccount", disable_swap_account);
4698 #endif