memcg: add mem_cgroup_zone_nr_pages()
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  */
19
20 #include <linux/res_counter.h>
21 #include <linux/memcontrol.h>
22 #include <linux/cgroup.h>
23 #include <linux/mm.h>
24 #include <linux/pagemap.h>
25 #include <linux/smp.h>
26 #include <linux/page-flags.h>
27 #include <linux/backing-dev.h>
28 #include <linux/bit_spinlock.h>
29 #include <linux/rcupdate.h>
30 #include <linux/mutex.h>
31 #include <linux/slab.h>
32 #include <linux/swap.h>
33 #include <linux/spinlock.h>
34 #include <linux/fs.h>
35 #include <linux/seq_file.h>
36 #include <linux/vmalloc.h>
37 #include <linux/mm_inline.h>
38 #include <linux/page_cgroup.h>
39 #include "internal.h"
40
41 #include <asm/uaccess.h>
42
43 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
44 #define MEM_CGROUP_RECLAIM_RETRIES      5
45
46 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
47 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 0 */
48 int do_swap_account __read_mostly;
49 static int really_do_swap_account __initdata = 1; /* for remember boot option*/
50 #else
51 #define do_swap_account         (0)
52 #endif
53
54
55 /*
56  * Statistics for memory cgroup.
57  */
58 enum mem_cgroup_stat_index {
59         /*
60          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
61          */
62         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
63         MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
64         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
65         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
66
67         MEM_CGROUP_STAT_NSTATS,
68 };
69
70 struct mem_cgroup_stat_cpu {
71         s64 count[MEM_CGROUP_STAT_NSTATS];
72 } ____cacheline_aligned_in_smp;
73
74 struct mem_cgroup_stat {
75         struct mem_cgroup_stat_cpu cpustat[0];
76 };
77
78 /*
79  * For accounting under irq disable, no need for increment preempt count.
80  */
81 static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
82                 enum mem_cgroup_stat_index idx, int val)
83 {
84         stat->count[idx] += val;
85 }
86
87 static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
88                 enum mem_cgroup_stat_index idx)
89 {
90         int cpu;
91         s64 ret = 0;
92         for_each_possible_cpu(cpu)
93                 ret += stat->cpustat[cpu].count[idx];
94         return ret;
95 }
96
97 /*
98  * per-zone information in memory controller.
99  */
100 struct mem_cgroup_per_zone {
101         /*
102          * spin_lock to protect the per cgroup LRU
103          */
104         struct list_head        lists[NR_LRU_LISTS];
105         unsigned long           count[NR_LRU_LISTS];
106 };
107 /* Macro for accessing counter */
108 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
109
110 struct mem_cgroup_per_node {
111         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
112 };
113
114 struct mem_cgroup_lru_info {
115         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
116 };
117
118 /*
119  * The memory controller data structure. The memory controller controls both
120  * page cache and RSS per cgroup. We would eventually like to provide
121  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
122  * to help the administrator determine what knobs to tune.
123  *
124  * TODO: Add a water mark for the memory controller. Reclaim will begin when
125  * we hit the water mark. May be even add a low water mark, such that
126  * no reclaim occurs from a cgroup at it's low water mark, this is
127  * a feature that will be implemented much later in the future.
128  */
129 struct mem_cgroup {
130         struct cgroup_subsys_state css;
131         /*
132          * the counter to account for memory usage
133          */
134         struct res_counter res;
135         /*
136          * the counter to account for mem+swap usage.
137          */
138         struct res_counter memsw;
139         /*
140          * Per cgroup active and inactive list, similar to the
141          * per zone LRU lists.
142          */
143         struct mem_cgroup_lru_info info;
144
145         int     prev_priority;  /* for recording reclaim priority */
146
147         /*
148          * While reclaiming in a hiearchy, we cache the last child we
149          * reclaimed from. Protected by cgroup_lock()
150          */
151         struct mem_cgroup *last_scanned_child;
152         /*
153          * Should the accounting and control be hierarchical, per subtree?
154          */
155         bool use_hierarchy;
156         unsigned long   last_oom_jiffies;
157         int             obsolete;
158         atomic_t        refcnt;
159
160         unsigned int inactive_ratio;
161
162         /*
163          * statistics. This must be placed at the end of memcg.
164          */
165         struct mem_cgroup_stat stat;
166 };
167
168 enum charge_type {
169         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
170         MEM_CGROUP_CHARGE_TYPE_MAPPED,
171         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
172         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
173         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
174         NR_CHARGE_TYPE,
175 };
176
177 /* only for here (for easy reading.) */
178 #define PCGF_CACHE      (1UL << PCG_CACHE)
179 #define PCGF_USED       (1UL << PCG_USED)
180 #define PCGF_LOCK       (1UL << PCG_LOCK)
181 static const unsigned long
182 pcg_default_flags[NR_CHARGE_TYPE] = {
183         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* File Cache */
184         PCGF_USED | PCGF_LOCK, /* Anon */
185         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
186         0, /* FORCE */
187 };
188
189 /* for encoding cft->private value on file */
190 #define _MEM                    (0)
191 #define _MEMSWAP                (1)
192 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
193 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
194 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
195
196 static void mem_cgroup_get(struct mem_cgroup *mem);
197 static void mem_cgroup_put(struct mem_cgroup *mem);
198
199 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
200                                          struct page_cgroup *pc,
201                                          bool charge)
202 {
203         int val = (charge)? 1 : -1;
204         struct mem_cgroup_stat *stat = &mem->stat;
205         struct mem_cgroup_stat_cpu *cpustat;
206         int cpu = get_cpu();
207
208         cpustat = &stat->cpustat[cpu];
209         if (PageCgroupCache(pc))
210                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
211         else
212                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
213
214         if (charge)
215                 __mem_cgroup_stat_add_safe(cpustat,
216                                 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
217         else
218                 __mem_cgroup_stat_add_safe(cpustat,
219                                 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
220         put_cpu();
221 }
222
223 static struct mem_cgroup_per_zone *
224 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
225 {
226         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
227 }
228
229 static struct mem_cgroup_per_zone *
230 page_cgroup_zoneinfo(struct page_cgroup *pc)
231 {
232         struct mem_cgroup *mem = pc->mem_cgroup;
233         int nid = page_cgroup_nid(pc);
234         int zid = page_cgroup_zid(pc);
235
236         if (!mem)
237                 return NULL;
238
239         return mem_cgroup_zoneinfo(mem, nid, zid);
240 }
241
242 static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
243                                         enum lru_list idx)
244 {
245         int nid, zid;
246         struct mem_cgroup_per_zone *mz;
247         u64 total = 0;
248
249         for_each_online_node(nid)
250                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
251                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
252                         total += MEM_CGROUP_ZSTAT(mz, idx);
253                 }
254         return total;
255 }
256
257 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
258 {
259         return container_of(cgroup_subsys_state(cont,
260                                 mem_cgroup_subsys_id), struct mem_cgroup,
261                                 css);
262 }
263
264 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
265 {
266         /*
267          * mm_update_next_owner() may clear mm->owner to NULL
268          * if it races with swapoff, page migration, etc.
269          * So this can be called with p == NULL.
270          */
271         if (unlikely(!p))
272                 return NULL;
273
274         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
275                                 struct mem_cgroup, css);
276 }
277
278 /*
279  * Following LRU functions are allowed to be used without PCG_LOCK.
280  * Operations are called by routine of global LRU independently from memcg.
281  * What we have to take care of here is validness of pc->mem_cgroup.
282  *
283  * Changes to pc->mem_cgroup happens when
284  * 1. charge
285  * 2. moving account
286  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
287  * It is added to LRU before charge.
288  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
289  * When moving account, the page is not on LRU. It's isolated.
290  */
291
292 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
293 {
294         struct page_cgroup *pc;
295         struct mem_cgroup *mem;
296         struct mem_cgroup_per_zone *mz;
297
298         if (mem_cgroup_disabled())
299                 return;
300         pc = lookup_page_cgroup(page);
301         /* can happen while we handle swapcache. */
302         if (list_empty(&pc->lru))
303                 return;
304         mz = page_cgroup_zoneinfo(pc);
305         mem = pc->mem_cgroup;
306         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
307         list_del_init(&pc->lru);
308         return;
309 }
310
311 void mem_cgroup_del_lru(struct page *page)
312 {
313         mem_cgroup_del_lru_list(page, page_lru(page));
314 }
315
316 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
317 {
318         struct mem_cgroup_per_zone *mz;
319         struct page_cgroup *pc;
320
321         if (mem_cgroup_disabled())
322                 return;
323
324         pc = lookup_page_cgroup(page);
325         smp_rmb();
326         /* unused page is not rotated. */
327         if (!PageCgroupUsed(pc))
328                 return;
329         mz = page_cgroup_zoneinfo(pc);
330         list_move(&pc->lru, &mz->lists[lru]);
331 }
332
333 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
334 {
335         struct page_cgroup *pc;
336         struct mem_cgroup_per_zone *mz;
337
338         if (mem_cgroup_disabled())
339                 return;
340         pc = lookup_page_cgroup(page);
341         /* barrier to sync with "charge" */
342         smp_rmb();
343         if (!PageCgroupUsed(pc))
344                 return;
345
346         mz = page_cgroup_zoneinfo(pc);
347         MEM_CGROUP_ZSTAT(mz, lru) += 1;
348         list_add(&pc->lru, &mz->lists[lru]);
349 }
350 /*
351  * To add swapcache into LRU. Be careful to all this function.
352  * zone->lru_lock shouldn't be held and irq must not be disabled.
353  */
354 static void mem_cgroup_lru_fixup(struct page *page)
355 {
356         if (!isolate_lru_page(page))
357                 putback_lru_page(page);
358 }
359
360 void mem_cgroup_move_lists(struct page *page,
361                            enum lru_list from, enum lru_list to)
362 {
363         if (mem_cgroup_disabled())
364                 return;
365         mem_cgroup_del_lru_list(page, from);
366         mem_cgroup_add_lru_list(page, to);
367 }
368
369 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
370 {
371         int ret;
372
373         task_lock(task);
374         ret = task->mm && mm_match_cgroup(task->mm, mem);
375         task_unlock(task);
376         return ret;
377 }
378
379 /*
380  * Calculate mapped_ratio under memory controller. This will be used in
381  * vmscan.c for deteremining we have to reclaim mapped pages.
382  */
383 int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
384 {
385         long total, rss;
386
387         /*
388          * usage is recorded in bytes. But, here, we assume the number of
389          * physical pages can be represented by "long" on any arch.
390          */
391         total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
392         rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
393         return (int)((rss * 100L) / total);
394 }
395
396 /*
397  * prev_priority control...this will be used in memory reclaim path.
398  */
399 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
400 {
401         return mem->prev_priority;
402 }
403
404 void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
405 {
406         if (priority < mem->prev_priority)
407                 mem->prev_priority = priority;
408 }
409
410 void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
411 {
412         mem->prev_priority = priority;
413 }
414
415 /*
416  * Calculate # of pages to be scanned in this priority/zone.
417  * See also vmscan.c
418  *
419  * priority starts from "DEF_PRIORITY" and decremented in each loop.
420  * (see include/linux/mmzone.h)
421  */
422
423 long mem_cgroup_calc_reclaim(struct mem_cgroup *mem, struct zone *zone,
424                                         int priority, enum lru_list lru)
425 {
426         long nr_pages;
427         int nid = zone->zone_pgdat->node_id;
428         int zid = zone_idx(zone);
429         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(mem, nid, zid);
430
431         nr_pages = MEM_CGROUP_ZSTAT(mz, lru);
432
433         return (nr_pages >> priority);
434 }
435
436 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
437 {
438         unsigned long active;
439         unsigned long inactive;
440
441         inactive = mem_cgroup_get_all_zonestat(memcg, LRU_INACTIVE_ANON);
442         active = mem_cgroup_get_all_zonestat(memcg, LRU_ACTIVE_ANON);
443
444         if (inactive * memcg->inactive_ratio < active)
445                 return 1;
446
447         return 0;
448 }
449
450 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
451                                        struct zone *zone,
452                                        enum lru_list lru)
453 {
454         int nid = zone->zone_pgdat->node_id;
455         int zid = zone_idx(zone);
456         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
457
458         return MEM_CGROUP_ZSTAT(mz, lru);
459 }
460
461 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
462                                         struct list_head *dst,
463                                         unsigned long *scanned, int order,
464                                         int mode, struct zone *z,
465                                         struct mem_cgroup *mem_cont,
466                                         int active, int file)
467 {
468         unsigned long nr_taken = 0;
469         struct page *page;
470         unsigned long scan;
471         LIST_HEAD(pc_list);
472         struct list_head *src;
473         struct page_cgroup *pc, *tmp;
474         int nid = z->zone_pgdat->node_id;
475         int zid = zone_idx(z);
476         struct mem_cgroup_per_zone *mz;
477         int lru = LRU_FILE * !!file + !!active;
478
479         BUG_ON(!mem_cont);
480         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
481         src = &mz->lists[lru];
482
483         scan = 0;
484         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
485                 if (scan >= nr_to_scan)
486                         break;
487
488                 page = pc->page;
489                 if (unlikely(!PageCgroupUsed(pc)))
490                         continue;
491                 if (unlikely(!PageLRU(page)))
492                         continue;
493
494                 scan++;
495                 if (__isolate_lru_page(page, mode, file) == 0) {
496                         list_move(&page->lru, dst);
497                         nr_taken++;
498                 }
499         }
500
501         *scanned = scan;
502         return nr_taken;
503 }
504
505 #define mem_cgroup_from_res_counter(counter, member)    \
506         container_of(counter, struct mem_cgroup, member)
507
508 /*
509  * This routine finds the DFS walk successor. This routine should be
510  * called with cgroup_mutex held
511  */
512 static struct mem_cgroup *
513 mem_cgroup_get_next_node(struct mem_cgroup *curr, struct mem_cgroup *root_mem)
514 {
515         struct cgroup *cgroup, *curr_cgroup, *root_cgroup;
516
517         curr_cgroup = curr->css.cgroup;
518         root_cgroup = root_mem->css.cgroup;
519
520         if (!list_empty(&curr_cgroup->children)) {
521                 /*
522                  * Walk down to children
523                  */
524                 mem_cgroup_put(curr);
525                 cgroup = list_entry(curr_cgroup->children.next,
526                                                 struct cgroup, sibling);
527                 curr = mem_cgroup_from_cont(cgroup);
528                 mem_cgroup_get(curr);
529                 goto done;
530         }
531
532 visit_parent:
533         if (curr_cgroup == root_cgroup) {
534                 mem_cgroup_put(curr);
535                 curr = root_mem;
536                 mem_cgroup_get(curr);
537                 goto done;
538         }
539
540         /*
541          * Goto next sibling
542          */
543         if (curr_cgroup->sibling.next != &curr_cgroup->parent->children) {
544                 mem_cgroup_put(curr);
545                 cgroup = list_entry(curr_cgroup->sibling.next, struct cgroup,
546                                                 sibling);
547                 curr = mem_cgroup_from_cont(cgroup);
548                 mem_cgroup_get(curr);
549                 goto done;
550         }
551
552         /*
553          * Go up to next parent and next parent's sibling if need be
554          */
555         curr_cgroup = curr_cgroup->parent;
556         goto visit_parent;
557
558 done:
559         root_mem->last_scanned_child = curr;
560         return curr;
561 }
562
563 /*
564  * Visit the first child (need not be the first child as per the ordering
565  * of the cgroup list, since we track last_scanned_child) of @mem and use
566  * that to reclaim free pages from.
567  */
568 static struct mem_cgroup *
569 mem_cgroup_get_first_node(struct mem_cgroup *root_mem)
570 {
571         struct cgroup *cgroup;
572         struct mem_cgroup *ret;
573         bool obsolete = (root_mem->last_scanned_child &&
574                                 root_mem->last_scanned_child->obsolete);
575
576         /*
577          * Scan all children under the mem_cgroup mem
578          */
579         cgroup_lock();
580         if (list_empty(&root_mem->css.cgroup->children)) {
581                 ret = root_mem;
582                 goto done;
583         }
584
585         if (!root_mem->last_scanned_child || obsolete) {
586
587                 if (obsolete)
588                         mem_cgroup_put(root_mem->last_scanned_child);
589
590                 cgroup = list_first_entry(&root_mem->css.cgroup->children,
591                                 struct cgroup, sibling);
592                 ret = mem_cgroup_from_cont(cgroup);
593                 mem_cgroup_get(ret);
594         } else
595                 ret = mem_cgroup_get_next_node(root_mem->last_scanned_child,
596                                                 root_mem);
597
598 done:
599         root_mem->last_scanned_child = ret;
600         cgroup_unlock();
601         return ret;
602 }
603
604 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
605 {
606         if (do_swap_account) {
607                 if (res_counter_check_under_limit(&mem->res) &&
608                         res_counter_check_under_limit(&mem->memsw))
609                         return true;
610         } else
611                 if (res_counter_check_under_limit(&mem->res))
612                         return true;
613         return false;
614 }
615
616 /*
617  * Dance down the hierarchy if needed to reclaim memory. We remember the
618  * last child we reclaimed from, so that we don't end up penalizing
619  * one child extensively based on its position in the children list.
620  *
621  * root_mem is the original ancestor that we've been reclaim from.
622  */
623 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
624                                                 gfp_t gfp_mask, bool noswap)
625 {
626         struct mem_cgroup *next_mem;
627         int ret = 0;
628
629         /*
630          * Reclaim unconditionally and don't check for return value.
631          * We need to reclaim in the current group and down the tree.
632          * One might think about checking for children before reclaiming,
633          * but there might be left over accounting, even after children
634          * have left.
635          */
636         ret = try_to_free_mem_cgroup_pages(root_mem, gfp_mask, noswap);
637         if (mem_cgroup_check_under_limit(root_mem))
638                 return 0;
639         if (!root_mem->use_hierarchy)
640                 return ret;
641
642         next_mem = mem_cgroup_get_first_node(root_mem);
643
644         while (next_mem != root_mem) {
645                 if (next_mem->obsolete) {
646                         mem_cgroup_put(next_mem);
647                         cgroup_lock();
648                         next_mem = mem_cgroup_get_first_node(root_mem);
649                         cgroup_unlock();
650                         continue;
651                 }
652                 ret = try_to_free_mem_cgroup_pages(next_mem, gfp_mask, noswap);
653                 if (mem_cgroup_check_under_limit(root_mem))
654                         return 0;
655                 cgroup_lock();
656                 next_mem = mem_cgroup_get_next_node(next_mem, root_mem);
657                 cgroup_unlock();
658         }
659         return ret;
660 }
661
662 bool mem_cgroup_oom_called(struct task_struct *task)
663 {
664         bool ret = false;
665         struct mem_cgroup *mem;
666         struct mm_struct *mm;
667
668         rcu_read_lock();
669         mm = task->mm;
670         if (!mm)
671                 mm = &init_mm;
672         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
673         if (mem && time_before(jiffies, mem->last_oom_jiffies + HZ/10))
674                 ret = true;
675         rcu_read_unlock();
676         return ret;
677 }
678 /*
679  * Unlike exported interface, "oom" parameter is added. if oom==true,
680  * oom-killer can be invoked.
681  */
682 static int __mem_cgroup_try_charge(struct mm_struct *mm,
683                         gfp_t gfp_mask, struct mem_cgroup **memcg,
684                         bool oom)
685 {
686         struct mem_cgroup *mem, *mem_over_limit;
687         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
688         struct res_counter *fail_res;
689
690         if (unlikely(test_thread_flag(TIF_MEMDIE))) {
691                 /* Don't account this! */
692                 *memcg = NULL;
693                 return 0;
694         }
695
696         /*
697          * We always charge the cgroup the mm_struct belongs to.
698          * The mm_struct's mem_cgroup changes on task migration if the
699          * thread group leader migrates. It's possible that mm is not
700          * set, if so charge the init_mm (happens for pagecache usage).
701          */
702         if (likely(!*memcg)) {
703                 rcu_read_lock();
704                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
705                 if (unlikely(!mem)) {
706                         rcu_read_unlock();
707                         return 0;
708                 }
709                 /*
710                  * For every charge from the cgroup, increment reference count
711                  */
712                 css_get(&mem->css);
713                 *memcg = mem;
714                 rcu_read_unlock();
715         } else {
716                 mem = *memcg;
717                 css_get(&mem->css);
718         }
719
720         while (1) {
721                 int ret;
722                 bool noswap = false;
723
724                 ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res);
725                 if (likely(!ret)) {
726                         if (!do_swap_account)
727                                 break;
728                         ret = res_counter_charge(&mem->memsw, PAGE_SIZE,
729                                                         &fail_res);
730                         if (likely(!ret))
731                                 break;
732                         /* mem+swap counter fails */
733                         res_counter_uncharge(&mem->res, PAGE_SIZE);
734                         noswap = true;
735                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
736                                                                         memsw);
737                 } else
738                         /* mem counter fails */
739                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
740                                                                         res);
741
742                 if (!(gfp_mask & __GFP_WAIT))
743                         goto nomem;
744
745                 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
746                                                         noswap);
747
748                 /*
749                  * try_to_free_mem_cgroup_pages() might not give us a full
750                  * picture of reclaim. Some pages are reclaimed and might be
751                  * moved to swap cache or just unmapped from the cgroup.
752                  * Check the limit again to see if the reclaim reduced the
753                  * current usage of the cgroup before giving up
754                  *
755                  */
756                 if (mem_cgroup_check_under_limit(mem_over_limit))
757                         continue;
758
759                 if (!nr_retries--) {
760                         if (oom) {
761                                 mem_cgroup_out_of_memory(mem_over_limit, gfp_mask);
762                                 mem_over_limit->last_oom_jiffies = jiffies;
763                         }
764                         goto nomem;
765                 }
766         }
767         return 0;
768 nomem:
769         css_put(&mem->css);
770         return -ENOMEM;
771 }
772
773 /**
774  * mem_cgroup_try_charge - get charge of PAGE_SIZE.
775  * @mm: an mm_struct which is charged against. (when *memcg is NULL)
776  * @gfp_mask: gfp_mask for reclaim.
777  * @memcg: a pointer to memory cgroup which is charged against.
778  *
779  * charge against memory cgroup pointed by *memcg. if *memcg == NULL, estimated
780  * memory cgroup from @mm is got and stored in *memcg.
781  *
782  * Returns 0 if success. -ENOMEM at failure.
783  * This call can invoke OOM-Killer.
784  */
785
786 int mem_cgroup_try_charge(struct mm_struct *mm,
787                           gfp_t mask, struct mem_cgroup **memcg)
788 {
789         return __mem_cgroup_try_charge(mm, mask, memcg, true);
790 }
791
792 /*
793  * commit a charge got by mem_cgroup_try_charge() and makes page_cgroup to be
794  * USED state. If already USED, uncharge and return.
795  */
796
797 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
798                                      struct page_cgroup *pc,
799                                      enum charge_type ctype)
800 {
801         /* try_charge() can return NULL to *memcg, taking care of it. */
802         if (!mem)
803                 return;
804
805         lock_page_cgroup(pc);
806         if (unlikely(PageCgroupUsed(pc))) {
807                 unlock_page_cgroup(pc);
808                 res_counter_uncharge(&mem->res, PAGE_SIZE);
809                 if (do_swap_account)
810                         res_counter_uncharge(&mem->memsw, PAGE_SIZE);
811                 css_put(&mem->css);
812                 return;
813         }
814         pc->mem_cgroup = mem;
815         smp_wmb();
816         pc->flags = pcg_default_flags[ctype];
817
818         mem_cgroup_charge_statistics(mem, pc, true);
819
820         unlock_page_cgroup(pc);
821 }
822
823 /**
824  * mem_cgroup_move_account - move account of the page
825  * @pc: page_cgroup of the page.
826  * @from: mem_cgroup which the page is moved from.
827  * @to: mem_cgroup which the page is moved to. @from != @to.
828  *
829  * The caller must confirm following.
830  * - page is not on LRU (isolate_page() is useful.)
831  *
832  * returns 0 at success,
833  * returns -EBUSY when lock is busy or "pc" is unstable.
834  *
835  * This function does "uncharge" from old cgroup but doesn't do "charge" to
836  * new cgroup. It should be done by a caller.
837  */
838
839 static int mem_cgroup_move_account(struct page_cgroup *pc,
840         struct mem_cgroup *from, struct mem_cgroup *to)
841 {
842         struct mem_cgroup_per_zone *from_mz, *to_mz;
843         int nid, zid;
844         int ret = -EBUSY;
845
846         VM_BUG_ON(from == to);
847         VM_BUG_ON(PageLRU(pc->page));
848
849         nid = page_cgroup_nid(pc);
850         zid = page_cgroup_zid(pc);
851         from_mz =  mem_cgroup_zoneinfo(from, nid, zid);
852         to_mz =  mem_cgroup_zoneinfo(to, nid, zid);
853
854         if (!trylock_page_cgroup(pc))
855                 return ret;
856
857         if (!PageCgroupUsed(pc))
858                 goto out;
859
860         if (pc->mem_cgroup != from)
861                 goto out;
862
863         css_put(&from->css);
864         res_counter_uncharge(&from->res, PAGE_SIZE);
865         mem_cgroup_charge_statistics(from, pc, false);
866         if (do_swap_account)
867                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
868         pc->mem_cgroup = to;
869         mem_cgroup_charge_statistics(to, pc, true);
870         css_get(&to->css);
871         ret = 0;
872 out:
873         unlock_page_cgroup(pc);
874         return ret;
875 }
876
877 /*
878  * move charges to its parent.
879  */
880
881 static int mem_cgroup_move_parent(struct page_cgroup *pc,
882                                   struct mem_cgroup *child,
883                                   gfp_t gfp_mask)
884 {
885         struct page *page = pc->page;
886         struct cgroup *cg = child->css.cgroup;
887         struct cgroup *pcg = cg->parent;
888         struct mem_cgroup *parent;
889         int ret;
890
891         /* Is ROOT ? */
892         if (!pcg)
893                 return -EINVAL;
894
895
896         parent = mem_cgroup_from_cont(pcg);
897
898
899         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
900         if (ret || !parent)
901                 return ret;
902
903         if (!get_page_unless_zero(page))
904                 return -EBUSY;
905
906         ret = isolate_lru_page(page);
907
908         if (ret)
909                 goto cancel;
910
911         ret = mem_cgroup_move_account(pc, child, parent);
912
913         /* drop extra refcnt by try_charge() (move_account increment one) */
914         css_put(&parent->css);
915         putback_lru_page(page);
916         if (!ret) {
917                 put_page(page);
918                 return 0;
919         }
920         /* uncharge if move fails */
921 cancel:
922         res_counter_uncharge(&parent->res, PAGE_SIZE);
923         if (do_swap_account)
924                 res_counter_uncharge(&parent->memsw, PAGE_SIZE);
925         put_page(page);
926         return ret;
927 }
928
929 /*
930  * Charge the memory controller for page usage.
931  * Return
932  * 0 if the charge was successful
933  * < 0 if the cgroup is over its limit
934  */
935 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
936                                 gfp_t gfp_mask, enum charge_type ctype,
937                                 struct mem_cgroup *memcg)
938 {
939         struct mem_cgroup *mem;
940         struct page_cgroup *pc;
941         int ret;
942
943         pc = lookup_page_cgroup(page);
944         /* can happen at boot */
945         if (unlikely(!pc))
946                 return 0;
947         prefetchw(pc);
948
949         mem = memcg;
950         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
951         if (ret || !mem)
952                 return ret;
953
954         __mem_cgroup_commit_charge(mem, pc, ctype);
955         return 0;
956 }
957
958 int mem_cgroup_newpage_charge(struct page *page,
959                               struct mm_struct *mm, gfp_t gfp_mask)
960 {
961         if (mem_cgroup_disabled())
962                 return 0;
963         if (PageCompound(page))
964                 return 0;
965         /*
966          * If already mapped, we don't have to account.
967          * If page cache, page->mapping has address_space.
968          * But page->mapping may have out-of-use anon_vma pointer,
969          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
970          * is NULL.
971          */
972         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
973                 return 0;
974         if (unlikely(!mm))
975                 mm = &init_mm;
976         return mem_cgroup_charge_common(page, mm, gfp_mask,
977                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
978 }
979
980 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
981                                 gfp_t gfp_mask)
982 {
983         if (mem_cgroup_disabled())
984                 return 0;
985         if (PageCompound(page))
986                 return 0;
987         /*
988          * Corner case handling. This is called from add_to_page_cache()
989          * in usual. But some FS (shmem) precharges this page before calling it
990          * and call add_to_page_cache() with GFP_NOWAIT.
991          *
992          * For GFP_NOWAIT case, the page may be pre-charged before calling
993          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
994          * charge twice. (It works but has to pay a bit larger cost.)
995          */
996         if (!(gfp_mask & __GFP_WAIT)) {
997                 struct page_cgroup *pc;
998
999
1000                 pc = lookup_page_cgroup(page);
1001                 if (!pc)
1002                         return 0;
1003                 lock_page_cgroup(pc);
1004                 if (PageCgroupUsed(pc)) {
1005                         unlock_page_cgroup(pc);
1006                         return 0;
1007                 }
1008                 unlock_page_cgroup(pc);
1009         }
1010
1011         if (unlikely(!mm))
1012                 mm = &init_mm;
1013
1014         if (page_is_file_cache(page))
1015                 return mem_cgroup_charge_common(page, mm, gfp_mask,
1016                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
1017         else
1018                 return mem_cgroup_charge_common(page, mm, gfp_mask,
1019                                 MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL);
1020 }
1021
1022 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
1023                                  struct page *page,
1024                                  gfp_t mask, struct mem_cgroup **ptr)
1025 {
1026         struct mem_cgroup *mem;
1027         swp_entry_t     ent;
1028
1029         if (mem_cgroup_disabled())
1030                 return 0;
1031
1032         if (!do_swap_account)
1033                 goto charge_cur_mm;
1034
1035         /*
1036          * A racing thread's fault, or swapoff, may have already updated
1037          * the pte, and even removed page from swap cache: return success
1038          * to go on to do_swap_page()'s pte_same() test, which should fail.
1039          */
1040         if (!PageSwapCache(page))
1041                 return 0;
1042
1043         ent.val = page_private(page);
1044
1045         mem = lookup_swap_cgroup(ent);
1046         if (!mem || mem->obsolete)
1047                 goto charge_cur_mm;
1048         *ptr = mem;
1049         return __mem_cgroup_try_charge(NULL, mask, ptr, true);
1050 charge_cur_mm:
1051         if (unlikely(!mm))
1052                 mm = &init_mm;
1053         return __mem_cgroup_try_charge(mm, mask, ptr, true);
1054 }
1055
1056 #ifdef CONFIG_SWAP
1057
1058 int mem_cgroup_cache_charge_swapin(struct page *page,
1059                         struct mm_struct *mm, gfp_t mask, bool locked)
1060 {
1061         int ret = 0;
1062
1063         if (mem_cgroup_disabled())
1064                 return 0;
1065         if (unlikely(!mm))
1066                 mm = &init_mm;
1067         if (!locked)
1068                 lock_page(page);
1069         /*
1070          * If not locked, the page can be dropped from SwapCache until
1071          * we reach here.
1072          */
1073         if (PageSwapCache(page)) {
1074                 struct mem_cgroup *mem = NULL;
1075                 swp_entry_t ent;
1076
1077                 ent.val = page_private(page);
1078                 if (do_swap_account) {
1079                         mem = lookup_swap_cgroup(ent);
1080                         if (mem && mem->obsolete)
1081                                 mem = NULL;
1082                         if (mem)
1083                                 mm = NULL;
1084                 }
1085                 ret = mem_cgroup_charge_common(page, mm, mask,
1086                                 MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
1087
1088                 if (!ret && do_swap_account) {
1089                         /* avoid double counting */
1090                         mem = swap_cgroup_record(ent, NULL);
1091                         if (mem) {
1092                                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1093                                 mem_cgroup_put(mem);
1094                         }
1095                 }
1096         }
1097         if (!locked)
1098                 unlock_page(page);
1099         /* add this page(page_cgroup) to the LRU we want. */
1100         mem_cgroup_lru_fixup(page);
1101
1102         return ret;
1103 }
1104 #endif
1105
1106 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
1107 {
1108         struct page_cgroup *pc;
1109
1110         if (mem_cgroup_disabled())
1111                 return;
1112         if (!ptr)
1113                 return;
1114         pc = lookup_page_cgroup(page);
1115         __mem_cgroup_commit_charge(ptr, pc, MEM_CGROUP_CHARGE_TYPE_MAPPED);
1116         /*
1117          * Now swap is on-memory. This means this page may be
1118          * counted both as mem and swap....double count.
1119          * Fix it by uncharging from memsw. This SwapCache is stable
1120          * because we're still under lock_page().
1121          */
1122         if (do_swap_account) {
1123                 swp_entry_t ent = {.val = page_private(page)};
1124                 struct mem_cgroup *memcg;
1125                 memcg = swap_cgroup_record(ent, NULL);
1126                 if (memcg) {
1127                         /* If memcg is obsolete, memcg can be != ptr */
1128                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1129                         mem_cgroup_put(memcg);
1130                 }
1131
1132         }
1133         /* add this page(page_cgroup) to the LRU we want. */
1134         mem_cgroup_lru_fixup(page);
1135 }
1136
1137 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
1138 {
1139         if (mem_cgroup_disabled())
1140                 return;
1141         if (!mem)
1142                 return;
1143         res_counter_uncharge(&mem->res, PAGE_SIZE);
1144         if (do_swap_account)
1145                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1146         css_put(&mem->css);
1147 }
1148
1149
1150 /*
1151  * uncharge if !page_mapped(page)
1152  */
1153 static struct mem_cgroup *
1154 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
1155 {
1156         struct page_cgroup *pc;
1157         struct mem_cgroup *mem = NULL;
1158         struct mem_cgroup_per_zone *mz;
1159
1160         if (mem_cgroup_disabled())
1161                 return NULL;
1162
1163         if (PageSwapCache(page))
1164                 return NULL;
1165
1166         /*
1167          * Check if our page_cgroup is valid
1168          */
1169         pc = lookup_page_cgroup(page);
1170         if (unlikely(!pc || !PageCgroupUsed(pc)))
1171                 return NULL;
1172
1173         lock_page_cgroup(pc);
1174
1175         mem = pc->mem_cgroup;
1176
1177         if (!PageCgroupUsed(pc))
1178                 goto unlock_out;
1179
1180         switch (ctype) {
1181         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
1182                 if (page_mapped(page))
1183                         goto unlock_out;
1184                 break;
1185         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
1186                 if (!PageAnon(page)) {  /* Shared memory */
1187                         if (page->mapping && !page_is_file_cache(page))
1188                                 goto unlock_out;
1189                 } else if (page_mapped(page)) /* Anon */
1190                                 goto unlock_out;
1191                 break;
1192         default:
1193                 break;
1194         }
1195
1196         res_counter_uncharge(&mem->res, PAGE_SIZE);
1197         if (do_swap_account && (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT))
1198                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1199
1200         mem_cgroup_charge_statistics(mem, pc, false);
1201         ClearPageCgroupUsed(pc);
1202
1203         mz = page_cgroup_zoneinfo(pc);
1204         unlock_page_cgroup(pc);
1205
1206         /* at swapout, this memcg will be accessed to record to swap */
1207         if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
1208                 css_put(&mem->css);
1209
1210         return mem;
1211
1212 unlock_out:
1213         unlock_page_cgroup(pc);
1214         return NULL;
1215 }
1216
1217 void mem_cgroup_uncharge_page(struct page *page)
1218 {
1219         /* early check. */
1220         if (page_mapped(page))
1221                 return;
1222         if (page->mapping && !PageAnon(page))
1223                 return;
1224         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
1225 }
1226
1227 void mem_cgroup_uncharge_cache_page(struct page *page)
1228 {
1229         VM_BUG_ON(page_mapped(page));
1230         VM_BUG_ON(page->mapping);
1231         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
1232 }
1233
1234 /*
1235  * called from __delete_from_swap_cache() and drop "page" account.
1236  * memcg information is recorded to swap_cgroup of "ent"
1237  */
1238 void mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent)
1239 {
1240         struct mem_cgroup *memcg;
1241
1242         memcg = __mem_cgroup_uncharge_common(page,
1243                                         MEM_CGROUP_CHARGE_TYPE_SWAPOUT);
1244         /* record memcg information */
1245         if (do_swap_account && memcg) {
1246                 swap_cgroup_record(ent, memcg);
1247                 mem_cgroup_get(memcg);
1248         }
1249         if (memcg)
1250                 css_put(&memcg->css);
1251 }
1252
1253 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1254 /*
1255  * called from swap_entry_free(). remove record in swap_cgroup and
1256  * uncharge "memsw" account.
1257  */
1258 void mem_cgroup_uncharge_swap(swp_entry_t ent)
1259 {
1260         struct mem_cgroup *memcg;
1261
1262         if (!do_swap_account)
1263                 return;
1264
1265         memcg = swap_cgroup_record(ent, NULL);
1266         if (memcg) {
1267                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1268                 mem_cgroup_put(memcg);
1269         }
1270 }
1271 #endif
1272
1273 /*
1274  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
1275  * page belongs to.
1276  */
1277 int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
1278 {
1279         struct page_cgroup *pc;
1280         struct mem_cgroup *mem = NULL;
1281         int ret = 0;
1282
1283         if (mem_cgroup_disabled())
1284                 return 0;
1285
1286         pc = lookup_page_cgroup(page);
1287         lock_page_cgroup(pc);
1288         if (PageCgroupUsed(pc)) {
1289                 mem = pc->mem_cgroup;
1290                 css_get(&mem->css);
1291         }
1292         unlock_page_cgroup(pc);
1293
1294         if (mem) {
1295                 ret = mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem);
1296                 css_put(&mem->css);
1297         }
1298         *ptr = mem;
1299         return ret;
1300 }
1301
1302 /* remove redundant charge if migration failed*/
1303 void mem_cgroup_end_migration(struct mem_cgroup *mem,
1304                 struct page *oldpage, struct page *newpage)
1305 {
1306         struct page *target, *unused;
1307         struct page_cgroup *pc;
1308         enum charge_type ctype;
1309
1310         if (!mem)
1311                 return;
1312
1313         /* at migration success, oldpage->mapping is NULL. */
1314         if (oldpage->mapping) {
1315                 target = oldpage;
1316                 unused = NULL;
1317         } else {
1318                 target = newpage;
1319                 unused = oldpage;
1320         }
1321
1322         if (PageAnon(target))
1323                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
1324         else if (page_is_file_cache(target))
1325                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
1326         else
1327                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
1328
1329         /* unused page is not on radix-tree now. */
1330         if (unused)
1331                 __mem_cgroup_uncharge_common(unused, ctype);
1332
1333         pc = lookup_page_cgroup(target);
1334         /*
1335          * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
1336          * So, double-counting is effectively avoided.
1337          */
1338         __mem_cgroup_commit_charge(mem, pc, ctype);
1339
1340         /*
1341          * Both of oldpage and newpage are still under lock_page().
1342          * Then, we don't have to care about race in radix-tree.
1343          * But we have to be careful that this page is unmapped or not.
1344          *
1345          * There is a case for !page_mapped(). At the start of
1346          * migration, oldpage was mapped. But now, it's zapped.
1347          * But we know *target* page is not freed/reused under us.
1348          * mem_cgroup_uncharge_page() does all necessary checks.
1349          */
1350         if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
1351                 mem_cgroup_uncharge_page(target);
1352 }
1353
1354 /*
1355  * A call to try to shrink memory usage under specified resource controller.
1356  * This is typically used for page reclaiming for shmem for reducing side
1357  * effect of page allocation from shmem, which is used by some mem_cgroup.
1358  */
1359 int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask)
1360 {
1361         struct mem_cgroup *mem;
1362         int progress = 0;
1363         int retry = MEM_CGROUP_RECLAIM_RETRIES;
1364
1365         if (mem_cgroup_disabled())
1366                 return 0;
1367         if (!mm)
1368                 return 0;
1369
1370         rcu_read_lock();
1371         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
1372         if (unlikely(!mem)) {
1373                 rcu_read_unlock();
1374                 return 0;
1375         }
1376         css_get(&mem->css);
1377         rcu_read_unlock();
1378
1379         do {
1380                 progress = try_to_free_mem_cgroup_pages(mem, gfp_mask, true);
1381                 progress += mem_cgroup_check_under_limit(mem);
1382         } while (!progress && --retry);
1383
1384         css_put(&mem->css);
1385         if (!retry)
1386                 return -ENOMEM;
1387         return 0;
1388 }
1389
1390 /*
1391  * The inactive anon list should be small enough that the VM never has to
1392  * do too much work, but large enough that each inactive page has a chance
1393  * to be referenced again before it is swapped out.
1394  *
1395  * this calculation is straightforward porting from
1396  * page_alloc.c::setup_per_zone_inactive_ratio().
1397  * it describe more detail.
1398  */
1399 static void mem_cgroup_set_inactive_ratio(struct mem_cgroup *memcg)
1400 {
1401         unsigned int gb, ratio;
1402
1403         gb = res_counter_read_u64(&memcg->res, RES_LIMIT) >> 30;
1404         if (gb)
1405                 ratio = int_sqrt(10 * gb);
1406         else
1407                 ratio = 1;
1408
1409         memcg->inactive_ratio = ratio;
1410
1411 }
1412
1413 static DEFINE_MUTEX(set_limit_mutex);
1414
1415 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
1416                                 unsigned long long val)
1417 {
1418
1419         int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
1420         int progress;
1421         u64 memswlimit;
1422         int ret = 0;
1423
1424         while (retry_count) {
1425                 if (signal_pending(current)) {
1426                         ret = -EINTR;
1427                         break;
1428                 }
1429                 /*
1430                  * Rather than hide all in some function, I do this in
1431                  * open coded manner. You see what this really does.
1432                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1433                  */
1434                 mutex_lock(&set_limit_mutex);
1435                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1436                 if (memswlimit < val) {
1437                         ret = -EINVAL;
1438                         mutex_unlock(&set_limit_mutex);
1439                         break;
1440                 }
1441                 ret = res_counter_set_limit(&memcg->res, val);
1442                 mutex_unlock(&set_limit_mutex);
1443
1444                 if (!ret)
1445                         break;
1446
1447                 progress = try_to_free_mem_cgroup_pages(memcg,
1448                                 GFP_KERNEL, false);
1449                 if (!progress)                  retry_count--;
1450         }
1451
1452         if (!ret)
1453                 mem_cgroup_set_inactive_ratio(memcg);
1454
1455         return ret;
1456 }
1457
1458 int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
1459                                 unsigned long long val)
1460 {
1461         int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
1462         u64 memlimit, oldusage, curusage;
1463         int ret;
1464
1465         if (!do_swap_account)
1466                 return -EINVAL;
1467
1468         while (retry_count) {
1469                 if (signal_pending(current)) {
1470                         ret = -EINTR;
1471                         break;
1472                 }
1473                 /*
1474                  * Rather than hide all in some function, I do this in
1475                  * open coded manner. You see what this really does.
1476                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1477                  */
1478                 mutex_lock(&set_limit_mutex);
1479                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1480                 if (memlimit > val) {
1481                         ret = -EINVAL;
1482                         mutex_unlock(&set_limit_mutex);
1483                         break;
1484                 }
1485                 ret = res_counter_set_limit(&memcg->memsw, val);
1486                 mutex_unlock(&set_limit_mutex);
1487
1488                 if (!ret)
1489                         break;
1490
1491                 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1492                 try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL, true);
1493                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1494                 if (curusage >= oldusage)
1495                         retry_count--;
1496         }
1497         return ret;
1498 }
1499
1500 /*
1501  * This routine traverse page_cgroup in given list and drop them all.
1502  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
1503  */
1504 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
1505                                 int node, int zid, enum lru_list lru)
1506 {
1507         struct zone *zone;
1508         struct mem_cgroup_per_zone *mz;
1509         struct page_cgroup *pc, *busy;
1510         unsigned long flags, loop;
1511         struct list_head *list;
1512         int ret = 0;
1513
1514         zone = &NODE_DATA(node)->node_zones[zid];
1515         mz = mem_cgroup_zoneinfo(mem, node, zid);
1516         list = &mz->lists[lru];
1517
1518         loop = MEM_CGROUP_ZSTAT(mz, lru);
1519         /* give some margin against EBUSY etc...*/
1520         loop += 256;
1521         busy = NULL;
1522         while (loop--) {
1523                 ret = 0;
1524                 spin_lock_irqsave(&zone->lru_lock, flags);
1525                 if (list_empty(list)) {
1526                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1527                         break;
1528                 }
1529                 pc = list_entry(list->prev, struct page_cgroup, lru);
1530                 if (busy == pc) {
1531                         list_move(&pc->lru, list);
1532                         busy = 0;
1533                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1534                         continue;
1535                 }
1536                 spin_unlock_irqrestore(&zone->lru_lock, flags);
1537
1538                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
1539                 if (ret == -ENOMEM)
1540                         break;
1541
1542                 if (ret == -EBUSY || ret == -EINVAL) {
1543                         /* found lock contention or "pc" is obsolete. */
1544                         busy = pc;
1545                         cond_resched();
1546                 } else
1547                         busy = NULL;
1548         }
1549
1550         if (!ret && !list_empty(list))
1551                 return -EBUSY;
1552         return ret;
1553 }
1554
1555 /*
1556  * make mem_cgroup's charge to be 0 if there is no task.
1557  * This enables deleting this mem_cgroup.
1558  */
1559 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
1560 {
1561         int ret;
1562         int node, zid, shrink;
1563         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1564         struct cgroup *cgrp = mem->css.cgroup;
1565
1566         css_get(&mem->css);
1567
1568         shrink = 0;
1569         /* should free all ? */
1570         if (free_all)
1571                 goto try_to_free;
1572 move_account:
1573         while (mem->res.usage > 0) {
1574                 ret = -EBUSY;
1575                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
1576                         goto out;
1577                 ret = -EINTR;
1578                 if (signal_pending(current))
1579                         goto out;
1580                 /* This is for making all *used* pages to be on LRU. */
1581                 lru_add_drain_all();
1582                 ret = 0;
1583                 for_each_node_state(node, N_POSSIBLE) {
1584                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
1585                                 enum lru_list l;
1586                                 for_each_lru(l) {
1587                                         ret = mem_cgroup_force_empty_list(mem,
1588                                                         node, zid, l);
1589                                         if (ret)
1590                                                 break;
1591                                 }
1592                         }
1593                         if (ret)
1594                                 break;
1595                 }
1596                 /* it seems parent cgroup doesn't have enough mem */
1597                 if (ret == -ENOMEM)
1598                         goto try_to_free;
1599                 cond_resched();
1600         }
1601         ret = 0;
1602 out:
1603         css_put(&mem->css);
1604         return ret;
1605
1606 try_to_free:
1607         /* returns EBUSY if there is a task or if we come here twice. */
1608         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
1609                 ret = -EBUSY;
1610                 goto out;
1611         }
1612         /* we call try-to-free pages for make this cgroup empty */
1613         lru_add_drain_all();
1614         /* try to free all pages in this cgroup */
1615         shrink = 1;
1616         while (nr_retries && mem->res.usage > 0) {
1617                 int progress;
1618
1619                 if (signal_pending(current)) {
1620                         ret = -EINTR;
1621                         goto out;
1622                 }
1623                 progress = try_to_free_mem_cgroup_pages(mem,
1624                                                   GFP_KERNEL, false);
1625                 if (!progress) {
1626                         nr_retries--;
1627                         /* maybe some writeback is necessary */
1628                         congestion_wait(WRITE, HZ/10);
1629                 }
1630
1631         }
1632         lru_add_drain();
1633         /* try move_account...there may be some *locked* pages. */
1634         if (mem->res.usage)
1635                 goto move_account;
1636         ret = 0;
1637         goto out;
1638 }
1639
1640 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
1641 {
1642         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
1643 }
1644
1645
1646 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
1647 {
1648         return mem_cgroup_from_cont(cont)->use_hierarchy;
1649 }
1650
1651 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
1652                                         u64 val)
1653 {
1654         int retval = 0;
1655         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1656         struct cgroup *parent = cont->parent;
1657         struct mem_cgroup *parent_mem = NULL;
1658
1659         if (parent)
1660                 parent_mem = mem_cgroup_from_cont(parent);
1661
1662         cgroup_lock();
1663         /*
1664          * If parent's use_hiearchy is set, we can't make any modifications
1665          * in the child subtrees. If it is unset, then the change can
1666          * occur, provided the current cgroup has no children.
1667          *
1668          * For the root cgroup, parent_mem is NULL, we allow value to be
1669          * set if there are no children.
1670          */
1671         if ((!parent_mem || !parent_mem->use_hierarchy) &&
1672                                 (val == 1 || val == 0)) {
1673                 if (list_empty(&cont->children))
1674                         mem->use_hierarchy = val;
1675                 else
1676                         retval = -EBUSY;
1677         } else
1678                 retval = -EINVAL;
1679         cgroup_unlock();
1680
1681         return retval;
1682 }
1683
1684 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
1685 {
1686         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1687         u64 val = 0;
1688         int type, name;
1689
1690         type = MEMFILE_TYPE(cft->private);
1691         name = MEMFILE_ATTR(cft->private);
1692         switch (type) {
1693         case _MEM:
1694                 val = res_counter_read_u64(&mem->res, name);
1695                 break;
1696         case _MEMSWAP:
1697                 if (do_swap_account)
1698                         val = res_counter_read_u64(&mem->memsw, name);
1699                 break;
1700         default:
1701                 BUG();
1702                 break;
1703         }
1704         return val;
1705 }
1706 /*
1707  * The user of this function is...
1708  * RES_LIMIT.
1709  */
1710 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
1711                             const char *buffer)
1712 {
1713         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
1714         int type, name;
1715         unsigned long long val;
1716         int ret;
1717
1718         type = MEMFILE_TYPE(cft->private);
1719         name = MEMFILE_ATTR(cft->private);
1720         switch (name) {
1721         case RES_LIMIT:
1722                 /* This function does all necessary parse...reuse it */
1723                 ret = res_counter_memparse_write_strategy(buffer, &val);
1724                 if (ret)
1725                         break;
1726                 if (type == _MEM)
1727                         ret = mem_cgroup_resize_limit(memcg, val);
1728                 else
1729                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
1730                 break;
1731         default:
1732                 ret = -EINVAL; /* should be BUG() ? */
1733                 break;
1734         }
1735         return ret;
1736 }
1737
1738 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
1739 {
1740         struct mem_cgroup *mem;
1741         int type, name;
1742
1743         mem = mem_cgroup_from_cont(cont);
1744         type = MEMFILE_TYPE(event);
1745         name = MEMFILE_ATTR(event);
1746         switch (name) {
1747         case RES_MAX_USAGE:
1748                 if (type == _MEM)
1749                         res_counter_reset_max(&mem->res);
1750                 else
1751                         res_counter_reset_max(&mem->memsw);
1752                 break;
1753         case RES_FAILCNT:
1754                 if (type == _MEM)
1755                         res_counter_reset_failcnt(&mem->res);
1756                 else
1757                         res_counter_reset_failcnt(&mem->memsw);
1758                 break;
1759         }
1760         return 0;
1761 }
1762
1763 static const struct mem_cgroup_stat_desc {
1764         const char *msg;
1765         u64 unit;
1766 } mem_cgroup_stat_desc[] = {
1767         [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
1768         [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
1769         [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, },
1770         [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, },
1771 };
1772
1773 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
1774                                  struct cgroup_map_cb *cb)
1775 {
1776         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
1777         struct mem_cgroup_stat *stat = &mem_cont->stat;
1778         int i;
1779
1780         for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
1781                 s64 val;
1782
1783                 val = mem_cgroup_read_stat(stat, i);
1784                 val *= mem_cgroup_stat_desc[i].unit;
1785                 cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
1786         }
1787         /* showing # of active pages */
1788         {
1789                 unsigned long active_anon, inactive_anon;
1790                 unsigned long active_file, inactive_file;
1791                 unsigned long unevictable;
1792
1793                 inactive_anon = mem_cgroup_get_all_zonestat(mem_cont,
1794                                                 LRU_INACTIVE_ANON);
1795                 active_anon = mem_cgroup_get_all_zonestat(mem_cont,
1796                                                 LRU_ACTIVE_ANON);
1797                 inactive_file = mem_cgroup_get_all_zonestat(mem_cont,
1798                                                 LRU_INACTIVE_FILE);
1799                 active_file = mem_cgroup_get_all_zonestat(mem_cont,
1800                                                 LRU_ACTIVE_FILE);
1801                 unevictable = mem_cgroup_get_all_zonestat(mem_cont,
1802                                                         LRU_UNEVICTABLE);
1803
1804                 cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE);
1805                 cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE);
1806                 cb->fill(cb, "active_file", (active_file) * PAGE_SIZE);
1807                 cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE);
1808                 cb->fill(cb, "unevictable", unevictable * PAGE_SIZE);
1809
1810         }
1811         return 0;
1812 }
1813
1814
1815 static struct cftype mem_cgroup_files[] = {
1816         {
1817                 .name = "usage_in_bytes",
1818                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
1819                 .read_u64 = mem_cgroup_read,
1820         },
1821         {
1822                 .name = "max_usage_in_bytes",
1823                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
1824                 .trigger = mem_cgroup_reset,
1825                 .read_u64 = mem_cgroup_read,
1826         },
1827         {
1828                 .name = "limit_in_bytes",
1829                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
1830                 .write_string = mem_cgroup_write,
1831                 .read_u64 = mem_cgroup_read,
1832         },
1833         {
1834                 .name = "failcnt",
1835                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
1836                 .trigger = mem_cgroup_reset,
1837                 .read_u64 = mem_cgroup_read,
1838         },
1839         {
1840                 .name = "stat",
1841                 .read_map = mem_control_stat_show,
1842         },
1843         {
1844                 .name = "force_empty",
1845                 .trigger = mem_cgroup_force_empty_write,
1846         },
1847         {
1848                 .name = "use_hierarchy",
1849                 .write_u64 = mem_cgroup_hierarchy_write,
1850                 .read_u64 = mem_cgroup_hierarchy_read,
1851         },
1852 };
1853
1854 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1855 static struct cftype memsw_cgroup_files[] = {
1856         {
1857                 .name = "memsw.usage_in_bytes",
1858                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
1859                 .read_u64 = mem_cgroup_read,
1860         },
1861         {
1862                 .name = "memsw.max_usage_in_bytes",
1863                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
1864                 .trigger = mem_cgroup_reset,
1865                 .read_u64 = mem_cgroup_read,
1866         },
1867         {
1868                 .name = "memsw.limit_in_bytes",
1869                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
1870                 .write_string = mem_cgroup_write,
1871                 .read_u64 = mem_cgroup_read,
1872         },
1873         {
1874                 .name = "memsw.failcnt",
1875                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
1876                 .trigger = mem_cgroup_reset,
1877                 .read_u64 = mem_cgroup_read,
1878         },
1879 };
1880
1881 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
1882 {
1883         if (!do_swap_account)
1884                 return 0;
1885         return cgroup_add_files(cont, ss, memsw_cgroup_files,
1886                                 ARRAY_SIZE(memsw_cgroup_files));
1887 };
1888 #else
1889 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
1890 {
1891         return 0;
1892 }
1893 #endif
1894
1895 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1896 {
1897         struct mem_cgroup_per_node *pn;
1898         struct mem_cgroup_per_zone *mz;
1899         enum lru_list l;
1900         int zone, tmp = node;
1901         /*
1902          * This routine is called against possible nodes.
1903          * But it's BUG to call kmalloc() against offline node.
1904          *
1905          * TODO: this routine can waste much memory for nodes which will
1906          *       never be onlined. It's better to use memory hotplug callback
1907          *       function.
1908          */
1909         if (!node_state(node, N_NORMAL_MEMORY))
1910                 tmp = -1;
1911         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
1912         if (!pn)
1913                 return 1;
1914
1915         mem->info.nodeinfo[node] = pn;
1916         memset(pn, 0, sizeof(*pn));
1917
1918         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
1919                 mz = &pn->zoneinfo[zone];
1920                 for_each_lru(l)
1921                         INIT_LIST_HEAD(&mz->lists[l]);
1922         }
1923         return 0;
1924 }
1925
1926 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1927 {
1928         kfree(mem->info.nodeinfo[node]);
1929 }
1930
1931 static int mem_cgroup_size(void)
1932 {
1933         int cpustat_size = nr_cpu_ids * sizeof(struct mem_cgroup_stat_cpu);
1934         return sizeof(struct mem_cgroup) + cpustat_size;
1935 }
1936
1937 static struct mem_cgroup *mem_cgroup_alloc(void)
1938 {
1939         struct mem_cgroup *mem;
1940         int size = mem_cgroup_size();
1941
1942         if (size < PAGE_SIZE)
1943                 mem = kmalloc(size, GFP_KERNEL);
1944         else
1945                 mem = vmalloc(size);
1946
1947         if (mem)
1948                 memset(mem, 0, size);
1949         return mem;
1950 }
1951
1952 /*
1953  * At destroying mem_cgroup, references from swap_cgroup can remain.
1954  * (scanning all at force_empty is too costly...)
1955  *
1956  * Instead of clearing all references at force_empty, we remember
1957  * the number of reference from swap_cgroup and free mem_cgroup when
1958  * it goes down to 0.
1959  *
1960  * When mem_cgroup is destroyed, mem->obsolete will be set to 0 and
1961  * entry which points to this memcg will be ignore at swapin.
1962  *
1963  * Removal of cgroup itself succeeds regardless of refs from swap.
1964  */
1965
1966 static void mem_cgroup_free(struct mem_cgroup *mem)
1967 {
1968         int node;
1969
1970         if (atomic_read(&mem->refcnt) > 0)
1971                 return;
1972
1973
1974         for_each_node_state(node, N_POSSIBLE)
1975                 free_mem_cgroup_per_zone_info(mem, node);
1976
1977         if (mem_cgroup_size() < PAGE_SIZE)
1978                 kfree(mem);
1979         else
1980                 vfree(mem);
1981 }
1982
1983 static void mem_cgroup_get(struct mem_cgroup *mem)
1984 {
1985         atomic_inc(&mem->refcnt);
1986 }
1987
1988 static void mem_cgroup_put(struct mem_cgroup *mem)
1989 {
1990         if (atomic_dec_and_test(&mem->refcnt)) {
1991                 if (!mem->obsolete)
1992                         return;
1993                 mem_cgroup_free(mem);
1994         }
1995 }
1996
1997
1998 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1999 static void __init enable_swap_cgroup(void)
2000 {
2001         if (!mem_cgroup_disabled() && really_do_swap_account)
2002                 do_swap_account = 1;
2003 }
2004 #else
2005 static void __init enable_swap_cgroup(void)
2006 {
2007 }
2008 #endif
2009
2010 static struct cgroup_subsys_state *
2011 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
2012 {
2013         struct mem_cgroup *mem, *parent;
2014         int node;
2015
2016         mem = mem_cgroup_alloc();
2017         if (!mem)
2018                 return ERR_PTR(-ENOMEM);
2019
2020         for_each_node_state(node, N_POSSIBLE)
2021                 if (alloc_mem_cgroup_per_zone_info(mem, node))
2022                         goto free_out;
2023         /* root ? */
2024         if (cont->parent == NULL) {
2025                 enable_swap_cgroup();
2026                 parent = NULL;
2027         } else {
2028                 parent = mem_cgroup_from_cont(cont->parent);
2029                 mem->use_hierarchy = parent->use_hierarchy;
2030         }
2031
2032         if (parent && parent->use_hierarchy) {
2033                 res_counter_init(&mem->res, &parent->res);
2034                 res_counter_init(&mem->memsw, &parent->memsw);
2035         } else {
2036                 res_counter_init(&mem->res, NULL);
2037                 res_counter_init(&mem->memsw, NULL);
2038         }
2039         mem_cgroup_set_inactive_ratio(mem);
2040         mem->last_scanned_child = NULL;
2041
2042         return &mem->css;
2043 free_out:
2044         for_each_node_state(node, N_POSSIBLE)
2045                 free_mem_cgroup_per_zone_info(mem, node);
2046         mem_cgroup_free(mem);
2047         return ERR_PTR(-ENOMEM);
2048 }
2049
2050 static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
2051                                         struct cgroup *cont)
2052 {
2053         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
2054         mem->obsolete = 1;
2055         mem_cgroup_force_empty(mem, false);
2056 }
2057
2058 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
2059                                 struct cgroup *cont)
2060 {
2061         mem_cgroup_free(mem_cgroup_from_cont(cont));
2062 }
2063
2064 static int mem_cgroup_populate(struct cgroup_subsys *ss,
2065                                 struct cgroup *cont)
2066 {
2067         int ret;
2068
2069         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
2070                                 ARRAY_SIZE(mem_cgroup_files));
2071
2072         if (!ret)
2073                 ret = register_memsw_files(cont, ss);
2074         return ret;
2075 }
2076
2077 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
2078                                 struct cgroup *cont,
2079                                 struct cgroup *old_cont,
2080                                 struct task_struct *p)
2081 {
2082         /*
2083          * FIXME: It's better to move charges of this process from old
2084          * memcg to new memcg. But it's just on TODO-List now.
2085          */
2086 }
2087
2088 struct cgroup_subsys mem_cgroup_subsys = {
2089         .name = "memory",
2090         .subsys_id = mem_cgroup_subsys_id,
2091         .create = mem_cgroup_create,
2092         .pre_destroy = mem_cgroup_pre_destroy,
2093         .destroy = mem_cgroup_destroy,
2094         .populate = mem_cgroup_populate,
2095         .attach = mem_cgroup_move_task,
2096         .early_init = 0,
2097 };
2098
2099 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2100
2101 static int __init disable_swap_account(char *s)
2102 {
2103         really_do_swap_account = 0;
2104         return 1;
2105 }
2106 __setup("noswapaccount", disable_swap_account);
2107 #endif