memcg: remove mem_cgroup_cal_reclaim()
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  */
19
20 #include <linux/res_counter.h>
21 #include <linux/memcontrol.h>
22 #include <linux/cgroup.h>
23 #include <linux/mm.h>
24 #include <linux/pagemap.h>
25 #include <linux/smp.h>
26 #include <linux/page-flags.h>
27 #include <linux/backing-dev.h>
28 #include <linux/bit_spinlock.h>
29 #include <linux/rcupdate.h>
30 #include <linux/mutex.h>
31 #include <linux/slab.h>
32 #include <linux/swap.h>
33 #include <linux/spinlock.h>
34 #include <linux/fs.h>
35 #include <linux/seq_file.h>
36 #include <linux/vmalloc.h>
37 #include <linux/mm_inline.h>
38 #include <linux/page_cgroup.h>
39 #include "internal.h"
40
41 #include <asm/uaccess.h>
42
43 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
44 #define MEM_CGROUP_RECLAIM_RETRIES      5
45
46 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
47 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 0 */
48 int do_swap_account __read_mostly;
49 static int really_do_swap_account __initdata = 1; /* for remember boot option*/
50 #else
51 #define do_swap_account         (0)
52 #endif
53
54
55 /*
56  * Statistics for memory cgroup.
57  */
58 enum mem_cgroup_stat_index {
59         /*
60          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
61          */
62         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
63         MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
64         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
65         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
66
67         MEM_CGROUP_STAT_NSTATS,
68 };
69
70 struct mem_cgroup_stat_cpu {
71         s64 count[MEM_CGROUP_STAT_NSTATS];
72 } ____cacheline_aligned_in_smp;
73
74 struct mem_cgroup_stat {
75         struct mem_cgroup_stat_cpu cpustat[0];
76 };
77
78 /*
79  * For accounting under irq disable, no need for increment preempt count.
80  */
81 static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
82                 enum mem_cgroup_stat_index idx, int val)
83 {
84         stat->count[idx] += val;
85 }
86
87 static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
88                 enum mem_cgroup_stat_index idx)
89 {
90         int cpu;
91         s64 ret = 0;
92         for_each_possible_cpu(cpu)
93                 ret += stat->cpustat[cpu].count[idx];
94         return ret;
95 }
96
97 /*
98  * per-zone information in memory controller.
99  */
100 struct mem_cgroup_per_zone {
101         /*
102          * spin_lock to protect the per cgroup LRU
103          */
104         struct list_head        lists[NR_LRU_LISTS];
105         unsigned long           count[NR_LRU_LISTS];
106
107         struct zone_reclaim_stat reclaim_stat;
108 };
109 /* Macro for accessing counter */
110 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
111
112 struct mem_cgroup_per_node {
113         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
114 };
115
116 struct mem_cgroup_lru_info {
117         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
118 };
119
120 /*
121  * The memory controller data structure. The memory controller controls both
122  * page cache and RSS per cgroup. We would eventually like to provide
123  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
124  * to help the administrator determine what knobs to tune.
125  *
126  * TODO: Add a water mark for the memory controller. Reclaim will begin when
127  * we hit the water mark. May be even add a low water mark, such that
128  * no reclaim occurs from a cgroup at it's low water mark, this is
129  * a feature that will be implemented much later in the future.
130  */
131 struct mem_cgroup {
132         struct cgroup_subsys_state css;
133         /*
134          * the counter to account for memory usage
135          */
136         struct res_counter res;
137         /*
138          * the counter to account for mem+swap usage.
139          */
140         struct res_counter memsw;
141         /*
142          * Per cgroup active and inactive list, similar to the
143          * per zone LRU lists.
144          */
145         struct mem_cgroup_lru_info info;
146
147         int     prev_priority;  /* for recording reclaim priority */
148
149         /*
150          * While reclaiming in a hiearchy, we cache the last child we
151          * reclaimed from. Protected by cgroup_lock()
152          */
153         struct mem_cgroup *last_scanned_child;
154         /*
155          * Should the accounting and control be hierarchical, per subtree?
156          */
157         bool use_hierarchy;
158         unsigned long   last_oom_jiffies;
159         int             obsolete;
160         atomic_t        refcnt;
161
162         unsigned int inactive_ratio;
163
164         /*
165          * statistics. This must be placed at the end of memcg.
166          */
167         struct mem_cgroup_stat stat;
168 };
169
170 enum charge_type {
171         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
172         MEM_CGROUP_CHARGE_TYPE_MAPPED,
173         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
174         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
175         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
176         NR_CHARGE_TYPE,
177 };
178
179 /* only for here (for easy reading.) */
180 #define PCGF_CACHE      (1UL << PCG_CACHE)
181 #define PCGF_USED       (1UL << PCG_USED)
182 #define PCGF_LOCK       (1UL << PCG_LOCK)
183 static const unsigned long
184 pcg_default_flags[NR_CHARGE_TYPE] = {
185         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* File Cache */
186         PCGF_USED | PCGF_LOCK, /* Anon */
187         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
188         0, /* FORCE */
189 };
190
191 /* for encoding cft->private value on file */
192 #define _MEM                    (0)
193 #define _MEMSWAP                (1)
194 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
195 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
196 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
197
198 static void mem_cgroup_get(struct mem_cgroup *mem);
199 static void mem_cgroup_put(struct mem_cgroup *mem);
200
201 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
202                                          struct page_cgroup *pc,
203                                          bool charge)
204 {
205         int val = (charge)? 1 : -1;
206         struct mem_cgroup_stat *stat = &mem->stat;
207         struct mem_cgroup_stat_cpu *cpustat;
208         int cpu = get_cpu();
209
210         cpustat = &stat->cpustat[cpu];
211         if (PageCgroupCache(pc))
212                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
213         else
214                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
215
216         if (charge)
217                 __mem_cgroup_stat_add_safe(cpustat,
218                                 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
219         else
220                 __mem_cgroup_stat_add_safe(cpustat,
221                                 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
222         put_cpu();
223 }
224
225 static struct mem_cgroup_per_zone *
226 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
227 {
228         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
229 }
230
231 static struct mem_cgroup_per_zone *
232 page_cgroup_zoneinfo(struct page_cgroup *pc)
233 {
234         struct mem_cgroup *mem = pc->mem_cgroup;
235         int nid = page_cgroup_nid(pc);
236         int zid = page_cgroup_zid(pc);
237
238         if (!mem)
239                 return NULL;
240
241         return mem_cgroup_zoneinfo(mem, nid, zid);
242 }
243
244 static unsigned long mem_cgroup_get_all_zonestat(struct mem_cgroup *mem,
245                                         enum lru_list idx)
246 {
247         int nid, zid;
248         struct mem_cgroup_per_zone *mz;
249         u64 total = 0;
250
251         for_each_online_node(nid)
252                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
253                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
254                         total += MEM_CGROUP_ZSTAT(mz, idx);
255                 }
256         return total;
257 }
258
259 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
260 {
261         return container_of(cgroup_subsys_state(cont,
262                                 mem_cgroup_subsys_id), struct mem_cgroup,
263                                 css);
264 }
265
266 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
267 {
268         /*
269          * mm_update_next_owner() may clear mm->owner to NULL
270          * if it races with swapoff, page migration, etc.
271          * So this can be called with p == NULL.
272          */
273         if (unlikely(!p))
274                 return NULL;
275
276         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
277                                 struct mem_cgroup, css);
278 }
279
280 /*
281  * Following LRU functions are allowed to be used without PCG_LOCK.
282  * Operations are called by routine of global LRU independently from memcg.
283  * What we have to take care of here is validness of pc->mem_cgroup.
284  *
285  * Changes to pc->mem_cgroup happens when
286  * 1. charge
287  * 2. moving account
288  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
289  * It is added to LRU before charge.
290  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
291  * When moving account, the page is not on LRU. It's isolated.
292  */
293
294 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
295 {
296         struct page_cgroup *pc;
297         struct mem_cgroup *mem;
298         struct mem_cgroup_per_zone *mz;
299
300         if (mem_cgroup_disabled())
301                 return;
302         pc = lookup_page_cgroup(page);
303         /* can happen while we handle swapcache. */
304         if (list_empty(&pc->lru))
305                 return;
306         mz = page_cgroup_zoneinfo(pc);
307         mem = pc->mem_cgroup;
308         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
309         list_del_init(&pc->lru);
310         return;
311 }
312
313 void mem_cgroup_del_lru(struct page *page)
314 {
315         mem_cgroup_del_lru_list(page, page_lru(page));
316 }
317
318 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
319 {
320         struct mem_cgroup_per_zone *mz;
321         struct page_cgroup *pc;
322
323         if (mem_cgroup_disabled())
324                 return;
325
326         pc = lookup_page_cgroup(page);
327         smp_rmb();
328         /* unused page is not rotated. */
329         if (!PageCgroupUsed(pc))
330                 return;
331         mz = page_cgroup_zoneinfo(pc);
332         list_move(&pc->lru, &mz->lists[lru]);
333 }
334
335 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
336 {
337         struct page_cgroup *pc;
338         struct mem_cgroup_per_zone *mz;
339
340         if (mem_cgroup_disabled())
341                 return;
342         pc = lookup_page_cgroup(page);
343         /* barrier to sync with "charge" */
344         smp_rmb();
345         if (!PageCgroupUsed(pc))
346                 return;
347
348         mz = page_cgroup_zoneinfo(pc);
349         MEM_CGROUP_ZSTAT(mz, lru) += 1;
350         list_add(&pc->lru, &mz->lists[lru]);
351 }
352 /*
353  * To add swapcache into LRU. Be careful to all this function.
354  * zone->lru_lock shouldn't be held and irq must not be disabled.
355  */
356 static void mem_cgroup_lru_fixup(struct page *page)
357 {
358         if (!isolate_lru_page(page))
359                 putback_lru_page(page);
360 }
361
362 void mem_cgroup_move_lists(struct page *page,
363                            enum lru_list from, enum lru_list to)
364 {
365         if (mem_cgroup_disabled())
366                 return;
367         mem_cgroup_del_lru_list(page, from);
368         mem_cgroup_add_lru_list(page, to);
369 }
370
371 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
372 {
373         int ret;
374
375         task_lock(task);
376         ret = task->mm && mm_match_cgroup(task->mm, mem);
377         task_unlock(task);
378         return ret;
379 }
380
381 /*
382  * Calculate mapped_ratio under memory controller. This will be used in
383  * vmscan.c for deteremining we have to reclaim mapped pages.
384  */
385 int mem_cgroup_calc_mapped_ratio(struct mem_cgroup *mem)
386 {
387         long total, rss;
388
389         /*
390          * usage is recorded in bytes. But, here, we assume the number of
391          * physical pages can be represented by "long" on any arch.
392          */
393         total = (long) (mem->res.usage >> PAGE_SHIFT) + 1L;
394         rss = (long)mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
395         return (int)((rss * 100L) / total);
396 }
397
398 /*
399  * prev_priority control...this will be used in memory reclaim path.
400  */
401 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
402 {
403         return mem->prev_priority;
404 }
405
406 void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
407 {
408         if (priority < mem->prev_priority)
409                 mem->prev_priority = priority;
410 }
411
412 void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
413 {
414         mem->prev_priority = priority;
415 }
416
417 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg, struct zone *zone)
418 {
419         unsigned long active;
420         unsigned long inactive;
421
422         inactive = mem_cgroup_get_all_zonestat(memcg, LRU_INACTIVE_ANON);
423         active = mem_cgroup_get_all_zonestat(memcg, LRU_ACTIVE_ANON);
424
425         if (inactive * memcg->inactive_ratio < active)
426                 return 1;
427
428         return 0;
429 }
430
431 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
432                                        struct zone *zone,
433                                        enum lru_list lru)
434 {
435         int nid = zone->zone_pgdat->node_id;
436         int zid = zone_idx(zone);
437         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
438
439         return MEM_CGROUP_ZSTAT(mz, lru);
440 }
441
442 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
443                                                       struct zone *zone)
444 {
445         int nid = zone->zone_pgdat->node_id;
446         int zid = zone_idx(zone);
447         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
448
449         return &mz->reclaim_stat;
450 }
451
452 struct zone_reclaim_stat *
453 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
454 {
455         struct page_cgroup *pc;
456         struct mem_cgroup_per_zone *mz;
457
458         if (mem_cgroup_disabled())
459                 return NULL;
460
461         pc = lookup_page_cgroup(page);
462         mz = page_cgroup_zoneinfo(pc);
463         if (!mz)
464                 return NULL;
465
466         return &mz->reclaim_stat;
467 }
468
469 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
470                                         struct list_head *dst,
471                                         unsigned long *scanned, int order,
472                                         int mode, struct zone *z,
473                                         struct mem_cgroup *mem_cont,
474                                         int active, int file)
475 {
476         unsigned long nr_taken = 0;
477         struct page *page;
478         unsigned long scan;
479         LIST_HEAD(pc_list);
480         struct list_head *src;
481         struct page_cgroup *pc, *tmp;
482         int nid = z->zone_pgdat->node_id;
483         int zid = zone_idx(z);
484         struct mem_cgroup_per_zone *mz;
485         int lru = LRU_FILE * !!file + !!active;
486
487         BUG_ON(!mem_cont);
488         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
489         src = &mz->lists[lru];
490
491         scan = 0;
492         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
493                 if (scan >= nr_to_scan)
494                         break;
495
496                 page = pc->page;
497                 if (unlikely(!PageCgroupUsed(pc)))
498                         continue;
499                 if (unlikely(!PageLRU(page)))
500                         continue;
501
502                 scan++;
503                 if (__isolate_lru_page(page, mode, file) == 0) {
504                         list_move(&page->lru, dst);
505                         nr_taken++;
506                 }
507         }
508
509         *scanned = scan;
510         return nr_taken;
511 }
512
513 #define mem_cgroup_from_res_counter(counter, member)    \
514         container_of(counter, struct mem_cgroup, member)
515
516 /*
517  * This routine finds the DFS walk successor. This routine should be
518  * called with cgroup_mutex held
519  */
520 static struct mem_cgroup *
521 mem_cgroup_get_next_node(struct mem_cgroup *curr, struct mem_cgroup *root_mem)
522 {
523         struct cgroup *cgroup, *curr_cgroup, *root_cgroup;
524
525         curr_cgroup = curr->css.cgroup;
526         root_cgroup = root_mem->css.cgroup;
527
528         if (!list_empty(&curr_cgroup->children)) {
529                 /*
530                  * Walk down to children
531                  */
532                 mem_cgroup_put(curr);
533                 cgroup = list_entry(curr_cgroup->children.next,
534                                                 struct cgroup, sibling);
535                 curr = mem_cgroup_from_cont(cgroup);
536                 mem_cgroup_get(curr);
537                 goto done;
538         }
539
540 visit_parent:
541         if (curr_cgroup == root_cgroup) {
542                 mem_cgroup_put(curr);
543                 curr = root_mem;
544                 mem_cgroup_get(curr);
545                 goto done;
546         }
547
548         /*
549          * Goto next sibling
550          */
551         if (curr_cgroup->sibling.next != &curr_cgroup->parent->children) {
552                 mem_cgroup_put(curr);
553                 cgroup = list_entry(curr_cgroup->sibling.next, struct cgroup,
554                                                 sibling);
555                 curr = mem_cgroup_from_cont(cgroup);
556                 mem_cgroup_get(curr);
557                 goto done;
558         }
559
560         /*
561          * Go up to next parent and next parent's sibling if need be
562          */
563         curr_cgroup = curr_cgroup->parent;
564         goto visit_parent;
565
566 done:
567         root_mem->last_scanned_child = curr;
568         return curr;
569 }
570
571 /*
572  * Visit the first child (need not be the first child as per the ordering
573  * of the cgroup list, since we track last_scanned_child) of @mem and use
574  * that to reclaim free pages from.
575  */
576 static struct mem_cgroup *
577 mem_cgroup_get_first_node(struct mem_cgroup *root_mem)
578 {
579         struct cgroup *cgroup;
580         struct mem_cgroup *ret;
581         bool obsolete = (root_mem->last_scanned_child &&
582                                 root_mem->last_scanned_child->obsolete);
583
584         /*
585          * Scan all children under the mem_cgroup mem
586          */
587         cgroup_lock();
588         if (list_empty(&root_mem->css.cgroup->children)) {
589                 ret = root_mem;
590                 goto done;
591         }
592
593         if (!root_mem->last_scanned_child || obsolete) {
594
595                 if (obsolete)
596                         mem_cgroup_put(root_mem->last_scanned_child);
597
598                 cgroup = list_first_entry(&root_mem->css.cgroup->children,
599                                 struct cgroup, sibling);
600                 ret = mem_cgroup_from_cont(cgroup);
601                 mem_cgroup_get(ret);
602         } else
603                 ret = mem_cgroup_get_next_node(root_mem->last_scanned_child,
604                                                 root_mem);
605
606 done:
607         root_mem->last_scanned_child = ret;
608         cgroup_unlock();
609         return ret;
610 }
611
612 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
613 {
614         if (do_swap_account) {
615                 if (res_counter_check_under_limit(&mem->res) &&
616                         res_counter_check_under_limit(&mem->memsw))
617                         return true;
618         } else
619                 if (res_counter_check_under_limit(&mem->res))
620                         return true;
621         return false;
622 }
623
624 /*
625  * Dance down the hierarchy if needed to reclaim memory. We remember the
626  * last child we reclaimed from, so that we don't end up penalizing
627  * one child extensively based on its position in the children list.
628  *
629  * root_mem is the original ancestor that we've been reclaim from.
630  */
631 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
632                                                 gfp_t gfp_mask, bool noswap)
633 {
634         struct mem_cgroup *next_mem;
635         int ret = 0;
636
637         /*
638          * Reclaim unconditionally and don't check for return value.
639          * We need to reclaim in the current group and down the tree.
640          * One might think about checking for children before reclaiming,
641          * but there might be left over accounting, even after children
642          * have left.
643          */
644         ret = try_to_free_mem_cgroup_pages(root_mem, gfp_mask, noswap);
645         if (mem_cgroup_check_under_limit(root_mem))
646                 return 0;
647         if (!root_mem->use_hierarchy)
648                 return ret;
649
650         next_mem = mem_cgroup_get_first_node(root_mem);
651
652         while (next_mem != root_mem) {
653                 if (next_mem->obsolete) {
654                         mem_cgroup_put(next_mem);
655                         cgroup_lock();
656                         next_mem = mem_cgroup_get_first_node(root_mem);
657                         cgroup_unlock();
658                         continue;
659                 }
660                 ret = try_to_free_mem_cgroup_pages(next_mem, gfp_mask, noswap);
661                 if (mem_cgroup_check_under_limit(root_mem))
662                         return 0;
663                 cgroup_lock();
664                 next_mem = mem_cgroup_get_next_node(next_mem, root_mem);
665                 cgroup_unlock();
666         }
667         return ret;
668 }
669
670 bool mem_cgroup_oom_called(struct task_struct *task)
671 {
672         bool ret = false;
673         struct mem_cgroup *mem;
674         struct mm_struct *mm;
675
676         rcu_read_lock();
677         mm = task->mm;
678         if (!mm)
679                 mm = &init_mm;
680         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
681         if (mem && time_before(jiffies, mem->last_oom_jiffies + HZ/10))
682                 ret = true;
683         rcu_read_unlock();
684         return ret;
685 }
686 /*
687  * Unlike exported interface, "oom" parameter is added. if oom==true,
688  * oom-killer can be invoked.
689  */
690 static int __mem_cgroup_try_charge(struct mm_struct *mm,
691                         gfp_t gfp_mask, struct mem_cgroup **memcg,
692                         bool oom)
693 {
694         struct mem_cgroup *mem, *mem_over_limit;
695         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
696         struct res_counter *fail_res;
697
698         if (unlikely(test_thread_flag(TIF_MEMDIE))) {
699                 /* Don't account this! */
700                 *memcg = NULL;
701                 return 0;
702         }
703
704         /*
705          * We always charge the cgroup the mm_struct belongs to.
706          * The mm_struct's mem_cgroup changes on task migration if the
707          * thread group leader migrates. It's possible that mm is not
708          * set, if so charge the init_mm (happens for pagecache usage).
709          */
710         if (likely(!*memcg)) {
711                 rcu_read_lock();
712                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
713                 if (unlikely(!mem)) {
714                         rcu_read_unlock();
715                         return 0;
716                 }
717                 /*
718                  * For every charge from the cgroup, increment reference count
719                  */
720                 css_get(&mem->css);
721                 *memcg = mem;
722                 rcu_read_unlock();
723         } else {
724                 mem = *memcg;
725                 css_get(&mem->css);
726         }
727
728         while (1) {
729                 int ret;
730                 bool noswap = false;
731
732                 ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res);
733                 if (likely(!ret)) {
734                         if (!do_swap_account)
735                                 break;
736                         ret = res_counter_charge(&mem->memsw, PAGE_SIZE,
737                                                         &fail_res);
738                         if (likely(!ret))
739                                 break;
740                         /* mem+swap counter fails */
741                         res_counter_uncharge(&mem->res, PAGE_SIZE);
742                         noswap = true;
743                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
744                                                                         memsw);
745                 } else
746                         /* mem counter fails */
747                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
748                                                                         res);
749
750                 if (!(gfp_mask & __GFP_WAIT))
751                         goto nomem;
752
753                 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
754                                                         noswap);
755
756                 /*
757                  * try_to_free_mem_cgroup_pages() might not give us a full
758                  * picture of reclaim. Some pages are reclaimed and might be
759                  * moved to swap cache or just unmapped from the cgroup.
760                  * Check the limit again to see if the reclaim reduced the
761                  * current usage of the cgroup before giving up
762                  *
763                  */
764                 if (mem_cgroup_check_under_limit(mem_over_limit))
765                         continue;
766
767                 if (!nr_retries--) {
768                         if (oom) {
769                                 mem_cgroup_out_of_memory(mem_over_limit, gfp_mask);
770                                 mem_over_limit->last_oom_jiffies = jiffies;
771                         }
772                         goto nomem;
773                 }
774         }
775         return 0;
776 nomem:
777         css_put(&mem->css);
778         return -ENOMEM;
779 }
780
781 /**
782  * mem_cgroup_try_charge - get charge of PAGE_SIZE.
783  * @mm: an mm_struct which is charged against. (when *memcg is NULL)
784  * @gfp_mask: gfp_mask for reclaim.
785  * @memcg: a pointer to memory cgroup which is charged against.
786  *
787  * charge against memory cgroup pointed by *memcg. if *memcg == NULL, estimated
788  * memory cgroup from @mm is got and stored in *memcg.
789  *
790  * Returns 0 if success. -ENOMEM at failure.
791  * This call can invoke OOM-Killer.
792  */
793
794 int mem_cgroup_try_charge(struct mm_struct *mm,
795                           gfp_t mask, struct mem_cgroup **memcg)
796 {
797         return __mem_cgroup_try_charge(mm, mask, memcg, true);
798 }
799
800 /*
801  * commit a charge got by mem_cgroup_try_charge() and makes page_cgroup to be
802  * USED state. If already USED, uncharge and return.
803  */
804
805 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
806                                      struct page_cgroup *pc,
807                                      enum charge_type ctype)
808 {
809         /* try_charge() can return NULL to *memcg, taking care of it. */
810         if (!mem)
811                 return;
812
813         lock_page_cgroup(pc);
814         if (unlikely(PageCgroupUsed(pc))) {
815                 unlock_page_cgroup(pc);
816                 res_counter_uncharge(&mem->res, PAGE_SIZE);
817                 if (do_swap_account)
818                         res_counter_uncharge(&mem->memsw, PAGE_SIZE);
819                 css_put(&mem->css);
820                 return;
821         }
822         pc->mem_cgroup = mem;
823         smp_wmb();
824         pc->flags = pcg_default_flags[ctype];
825
826         mem_cgroup_charge_statistics(mem, pc, true);
827
828         unlock_page_cgroup(pc);
829 }
830
831 /**
832  * mem_cgroup_move_account - move account of the page
833  * @pc: page_cgroup of the page.
834  * @from: mem_cgroup which the page is moved from.
835  * @to: mem_cgroup which the page is moved to. @from != @to.
836  *
837  * The caller must confirm following.
838  * - page is not on LRU (isolate_page() is useful.)
839  *
840  * returns 0 at success,
841  * returns -EBUSY when lock is busy or "pc" is unstable.
842  *
843  * This function does "uncharge" from old cgroup but doesn't do "charge" to
844  * new cgroup. It should be done by a caller.
845  */
846
847 static int mem_cgroup_move_account(struct page_cgroup *pc,
848         struct mem_cgroup *from, struct mem_cgroup *to)
849 {
850         struct mem_cgroup_per_zone *from_mz, *to_mz;
851         int nid, zid;
852         int ret = -EBUSY;
853
854         VM_BUG_ON(from == to);
855         VM_BUG_ON(PageLRU(pc->page));
856
857         nid = page_cgroup_nid(pc);
858         zid = page_cgroup_zid(pc);
859         from_mz =  mem_cgroup_zoneinfo(from, nid, zid);
860         to_mz =  mem_cgroup_zoneinfo(to, nid, zid);
861
862         if (!trylock_page_cgroup(pc))
863                 return ret;
864
865         if (!PageCgroupUsed(pc))
866                 goto out;
867
868         if (pc->mem_cgroup != from)
869                 goto out;
870
871         css_put(&from->css);
872         res_counter_uncharge(&from->res, PAGE_SIZE);
873         mem_cgroup_charge_statistics(from, pc, false);
874         if (do_swap_account)
875                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
876         pc->mem_cgroup = to;
877         mem_cgroup_charge_statistics(to, pc, true);
878         css_get(&to->css);
879         ret = 0;
880 out:
881         unlock_page_cgroup(pc);
882         return ret;
883 }
884
885 /*
886  * move charges to its parent.
887  */
888
889 static int mem_cgroup_move_parent(struct page_cgroup *pc,
890                                   struct mem_cgroup *child,
891                                   gfp_t gfp_mask)
892 {
893         struct page *page = pc->page;
894         struct cgroup *cg = child->css.cgroup;
895         struct cgroup *pcg = cg->parent;
896         struct mem_cgroup *parent;
897         int ret;
898
899         /* Is ROOT ? */
900         if (!pcg)
901                 return -EINVAL;
902
903
904         parent = mem_cgroup_from_cont(pcg);
905
906
907         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
908         if (ret || !parent)
909                 return ret;
910
911         if (!get_page_unless_zero(page))
912                 return -EBUSY;
913
914         ret = isolate_lru_page(page);
915
916         if (ret)
917                 goto cancel;
918
919         ret = mem_cgroup_move_account(pc, child, parent);
920
921         /* drop extra refcnt by try_charge() (move_account increment one) */
922         css_put(&parent->css);
923         putback_lru_page(page);
924         if (!ret) {
925                 put_page(page);
926                 return 0;
927         }
928         /* uncharge if move fails */
929 cancel:
930         res_counter_uncharge(&parent->res, PAGE_SIZE);
931         if (do_swap_account)
932                 res_counter_uncharge(&parent->memsw, PAGE_SIZE);
933         put_page(page);
934         return ret;
935 }
936
937 /*
938  * Charge the memory controller for page usage.
939  * Return
940  * 0 if the charge was successful
941  * < 0 if the cgroup is over its limit
942  */
943 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
944                                 gfp_t gfp_mask, enum charge_type ctype,
945                                 struct mem_cgroup *memcg)
946 {
947         struct mem_cgroup *mem;
948         struct page_cgroup *pc;
949         int ret;
950
951         pc = lookup_page_cgroup(page);
952         /* can happen at boot */
953         if (unlikely(!pc))
954                 return 0;
955         prefetchw(pc);
956
957         mem = memcg;
958         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
959         if (ret || !mem)
960                 return ret;
961
962         __mem_cgroup_commit_charge(mem, pc, ctype);
963         return 0;
964 }
965
966 int mem_cgroup_newpage_charge(struct page *page,
967                               struct mm_struct *mm, gfp_t gfp_mask)
968 {
969         if (mem_cgroup_disabled())
970                 return 0;
971         if (PageCompound(page))
972                 return 0;
973         /*
974          * If already mapped, we don't have to account.
975          * If page cache, page->mapping has address_space.
976          * But page->mapping may have out-of-use anon_vma pointer,
977          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
978          * is NULL.
979          */
980         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
981                 return 0;
982         if (unlikely(!mm))
983                 mm = &init_mm;
984         return mem_cgroup_charge_common(page, mm, gfp_mask,
985                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
986 }
987
988 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
989                                 gfp_t gfp_mask)
990 {
991         if (mem_cgroup_disabled())
992                 return 0;
993         if (PageCompound(page))
994                 return 0;
995         /*
996          * Corner case handling. This is called from add_to_page_cache()
997          * in usual. But some FS (shmem) precharges this page before calling it
998          * and call add_to_page_cache() with GFP_NOWAIT.
999          *
1000          * For GFP_NOWAIT case, the page may be pre-charged before calling
1001          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
1002          * charge twice. (It works but has to pay a bit larger cost.)
1003          */
1004         if (!(gfp_mask & __GFP_WAIT)) {
1005                 struct page_cgroup *pc;
1006
1007
1008                 pc = lookup_page_cgroup(page);
1009                 if (!pc)
1010                         return 0;
1011                 lock_page_cgroup(pc);
1012                 if (PageCgroupUsed(pc)) {
1013                         unlock_page_cgroup(pc);
1014                         return 0;
1015                 }
1016                 unlock_page_cgroup(pc);
1017         }
1018
1019         if (unlikely(!mm))
1020                 mm = &init_mm;
1021
1022         if (page_is_file_cache(page))
1023                 return mem_cgroup_charge_common(page, mm, gfp_mask,
1024                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
1025         else
1026                 return mem_cgroup_charge_common(page, mm, gfp_mask,
1027                                 MEM_CGROUP_CHARGE_TYPE_SHMEM, NULL);
1028 }
1029
1030 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
1031                                  struct page *page,
1032                                  gfp_t mask, struct mem_cgroup **ptr)
1033 {
1034         struct mem_cgroup *mem;
1035         swp_entry_t     ent;
1036
1037         if (mem_cgroup_disabled())
1038                 return 0;
1039
1040         if (!do_swap_account)
1041                 goto charge_cur_mm;
1042
1043         /*
1044          * A racing thread's fault, or swapoff, may have already updated
1045          * the pte, and even removed page from swap cache: return success
1046          * to go on to do_swap_page()'s pte_same() test, which should fail.
1047          */
1048         if (!PageSwapCache(page))
1049                 return 0;
1050
1051         ent.val = page_private(page);
1052
1053         mem = lookup_swap_cgroup(ent);
1054         if (!mem || mem->obsolete)
1055                 goto charge_cur_mm;
1056         *ptr = mem;
1057         return __mem_cgroup_try_charge(NULL, mask, ptr, true);
1058 charge_cur_mm:
1059         if (unlikely(!mm))
1060                 mm = &init_mm;
1061         return __mem_cgroup_try_charge(mm, mask, ptr, true);
1062 }
1063
1064 #ifdef CONFIG_SWAP
1065
1066 int mem_cgroup_cache_charge_swapin(struct page *page,
1067                         struct mm_struct *mm, gfp_t mask, bool locked)
1068 {
1069         int ret = 0;
1070
1071         if (mem_cgroup_disabled())
1072                 return 0;
1073         if (unlikely(!mm))
1074                 mm = &init_mm;
1075         if (!locked)
1076                 lock_page(page);
1077         /*
1078          * If not locked, the page can be dropped from SwapCache until
1079          * we reach here.
1080          */
1081         if (PageSwapCache(page)) {
1082                 struct mem_cgroup *mem = NULL;
1083                 swp_entry_t ent;
1084
1085                 ent.val = page_private(page);
1086                 if (do_swap_account) {
1087                         mem = lookup_swap_cgroup(ent);
1088                         if (mem && mem->obsolete)
1089                                 mem = NULL;
1090                         if (mem)
1091                                 mm = NULL;
1092                 }
1093                 ret = mem_cgroup_charge_common(page, mm, mask,
1094                                 MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
1095
1096                 if (!ret && do_swap_account) {
1097                         /* avoid double counting */
1098                         mem = swap_cgroup_record(ent, NULL);
1099                         if (mem) {
1100                                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1101                                 mem_cgroup_put(mem);
1102                         }
1103                 }
1104         }
1105         if (!locked)
1106                 unlock_page(page);
1107         /* add this page(page_cgroup) to the LRU we want. */
1108         mem_cgroup_lru_fixup(page);
1109
1110         return ret;
1111 }
1112 #endif
1113
1114 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
1115 {
1116         struct page_cgroup *pc;
1117
1118         if (mem_cgroup_disabled())
1119                 return;
1120         if (!ptr)
1121                 return;
1122         pc = lookup_page_cgroup(page);
1123         __mem_cgroup_commit_charge(ptr, pc, MEM_CGROUP_CHARGE_TYPE_MAPPED);
1124         /*
1125          * Now swap is on-memory. This means this page may be
1126          * counted both as mem and swap....double count.
1127          * Fix it by uncharging from memsw. This SwapCache is stable
1128          * because we're still under lock_page().
1129          */
1130         if (do_swap_account) {
1131                 swp_entry_t ent = {.val = page_private(page)};
1132                 struct mem_cgroup *memcg;
1133                 memcg = swap_cgroup_record(ent, NULL);
1134                 if (memcg) {
1135                         /* If memcg is obsolete, memcg can be != ptr */
1136                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1137                         mem_cgroup_put(memcg);
1138                 }
1139
1140         }
1141         /* add this page(page_cgroup) to the LRU we want. */
1142         mem_cgroup_lru_fixup(page);
1143 }
1144
1145 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
1146 {
1147         if (mem_cgroup_disabled())
1148                 return;
1149         if (!mem)
1150                 return;
1151         res_counter_uncharge(&mem->res, PAGE_SIZE);
1152         if (do_swap_account)
1153                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1154         css_put(&mem->css);
1155 }
1156
1157
1158 /*
1159  * uncharge if !page_mapped(page)
1160  */
1161 static struct mem_cgroup *
1162 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
1163 {
1164         struct page_cgroup *pc;
1165         struct mem_cgroup *mem = NULL;
1166         struct mem_cgroup_per_zone *mz;
1167
1168         if (mem_cgroup_disabled())
1169                 return NULL;
1170
1171         if (PageSwapCache(page))
1172                 return NULL;
1173
1174         /*
1175          * Check if our page_cgroup is valid
1176          */
1177         pc = lookup_page_cgroup(page);
1178         if (unlikely(!pc || !PageCgroupUsed(pc)))
1179                 return NULL;
1180
1181         lock_page_cgroup(pc);
1182
1183         mem = pc->mem_cgroup;
1184
1185         if (!PageCgroupUsed(pc))
1186                 goto unlock_out;
1187
1188         switch (ctype) {
1189         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
1190                 if (page_mapped(page))
1191                         goto unlock_out;
1192                 break;
1193         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
1194                 if (!PageAnon(page)) {  /* Shared memory */
1195                         if (page->mapping && !page_is_file_cache(page))
1196                                 goto unlock_out;
1197                 } else if (page_mapped(page)) /* Anon */
1198                                 goto unlock_out;
1199                 break;
1200         default:
1201                 break;
1202         }
1203
1204         res_counter_uncharge(&mem->res, PAGE_SIZE);
1205         if (do_swap_account && (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT))
1206                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1207
1208         mem_cgroup_charge_statistics(mem, pc, false);
1209         ClearPageCgroupUsed(pc);
1210
1211         mz = page_cgroup_zoneinfo(pc);
1212         unlock_page_cgroup(pc);
1213
1214         /* at swapout, this memcg will be accessed to record to swap */
1215         if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
1216                 css_put(&mem->css);
1217
1218         return mem;
1219
1220 unlock_out:
1221         unlock_page_cgroup(pc);
1222         return NULL;
1223 }
1224
1225 void mem_cgroup_uncharge_page(struct page *page)
1226 {
1227         /* early check. */
1228         if (page_mapped(page))
1229                 return;
1230         if (page->mapping && !PageAnon(page))
1231                 return;
1232         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
1233 }
1234
1235 void mem_cgroup_uncharge_cache_page(struct page *page)
1236 {
1237         VM_BUG_ON(page_mapped(page));
1238         VM_BUG_ON(page->mapping);
1239         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
1240 }
1241
1242 /*
1243  * called from __delete_from_swap_cache() and drop "page" account.
1244  * memcg information is recorded to swap_cgroup of "ent"
1245  */
1246 void mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent)
1247 {
1248         struct mem_cgroup *memcg;
1249
1250         memcg = __mem_cgroup_uncharge_common(page,
1251                                         MEM_CGROUP_CHARGE_TYPE_SWAPOUT);
1252         /* record memcg information */
1253         if (do_swap_account && memcg) {
1254                 swap_cgroup_record(ent, memcg);
1255                 mem_cgroup_get(memcg);
1256         }
1257         if (memcg)
1258                 css_put(&memcg->css);
1259 }
1260
1261 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1262 /*
1263  * called from swap_entry_free(). remove record in swap_cgroup and
1264  * uncharge "memsw" account.
1265  */
1266 void mem_cgroup_uncharge_swap(swp_entry_t ent)
1267 {
1268         struct mem_cgroup *memcg;
1269
1270         if (!do_swap_account)
1271                 return;
1272
1273         memcg = swap_cgroup_record(ent, NULL);
1274         if (memcg) {
1275                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1276                 mem_cgroup_put(memcg);
1277         }
1278 }
1279 #endif
1280
1281 /*
1282  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
1283  * page belongs to.
1284  */
1285 int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
1286 {
1287         struct page_cgroup *pc;
1288         struct mem_cgroup *mem = NULL;
1289         int ret = 0;
1290
1291         if (mem_cgroup_disabled())
1292                 return 0;
1293
1294         pc = lookup_page_cgroup(page);
1295         lock_page_cgroup(pc);
1296         if (PageCgroupUsed(pc)) {
1297                 mem = pc->mem_cgroup;
1298                 css_get(&mem->css);
1299         }
1300         unlock_page_cgroup(pc);
1301
1302         if (mem) {
1303                 ret = mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem);
1304                 css_put(&mem->css);
1305         }
1306         *ptr = mem;
1307         return ret;
1308 }
1309
1310 /* remove redundant charge if migration failed*/
1311 void mem_cgroup_end_migration(struct mem_cgroup *mem,
1312                 struct page *oldpage, struct page *newpage)
1313 {
1314         struct page *target, *unused;
1315         struct page_cgroup *pc;
1316         enum charge_type ctype;
1317
1318         if (!mem)
1319                 return;
1320
1321         /* at migration success, oldpage->mapping is NULL. */
1322         if (oldpage->mapping) {
1323                 target = oldpage;
1324                 unused = NULL;
1325         } else {
1326                 target = newpage;
1327                 unused = oldpage;
1328         }
1329
1330         if (PageAnon(target))
1331                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
1332         else if (page_is_file_cache(target))
1333                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
1334         else
1335                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
1336
1337         /* unused page is not on radix-tree now. */
1338         if (unused)
1339                 __mem_cgroup_uncharge_common(unused, ctype);
1340
1341         pc = lookup_page_cgroup(target);
1342         /*
1343          * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
1344          * So, double-counting is effectively avoided.
1345          */
1346         __mem_cgroup_commit_charge(mem, pc, ctype);
1347
1348         /*
1349          * Both of oldpage and newpage are still under lock_page().
1350          * Then, we don't have to care about race in radix-tree.
1351          * But we have to be careful that this page is unmapped or not.
1352          *
1353          * There is a case for !page_mapped(). At the start of
1354          * migration, oldpage was mapped. But now, it's zapped.
1355          * But we know *target* page is not freed/reused under us.
1356          * mem_cgroup_uncharge_page() does all necessary checks.
1357          */
1358         if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
1359                 mem_cgroup_uncharge_page(target);
1360 }
1361
1362 /*
1363  * A call to try to shrink memory usage under specified resource controller.
1364  * This is typically used for page reclaiming for shmem for reducing side
1365  * effect of page allocation from shmem, which is used by some mem_cgroup.
1366  */
1367 int mem_cgroup_shrink_usage(struct mm_struct *mm, gfp_t gfp_mask)
1368 {
1369         struct mem_cgroup *mem;
1370         int progress = 0;
1371         int retry = MEM_CGROUP_RECLAIM_RETRIES;
1372
1373         if (mem_cgroup_disabled())
1374                 return 0;
1375         if (!mm)
1376                 return 0;
1377
1378         rcu_read_lock();
1379         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
1380         if (unlikely(!mem)) {
1381                 rcu_read_unlock();
1382                 return 0;
1383         }
1384         css_get(&mem->css);
1385         rcu_read_unlock();
1386
1387         do {
1388                 progress = try_to_free_mem_cgroup_pages(mem, gfp_mask, true);
1389                 progress += mem_cgroup_check_under_limit(mem);
1390         } while (!progress && --retry);
1391
1392         css_put(&mem->css);
1393         if (!retry)
1394                 return -ENOMEM;
1395         return 0;
1396 }
1397
1398 /*
1399  * The inactive anon list should be small enough that the VM never has to
1400  * do too much work, but large enough that each inactive page has a chance
1401  * to be referenced again before it is swapped out.
1402  *
1403  * this calculation is straightforward porting from
1404  * page_alloc.c::setup_per_zone_inactive_ratio().
1405  * it describe more detail.
1406  */
1407 static void mem_cgroup_set_inactive_ratio(struct mem_cgroup *memcg)
1408 {
1409         unsigned int gb, ratio;
1410
1411         gb = res_counter_read_u64(&memcg->res, RES_LIMIT) >> 30;
1412         if (gb)
1413                 ratio = int_sqrt(10 * gb);
1414         else
1415                 ratio = 1;
1416
1417         memcg->inactive_ratio = ratio;
1418
1419 }
1420
1421 static DEFINE_MUTEX(set_limit_mutex);
1422
1423 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
1424                                 unsigned long long val)
1425 {
1426
1427         int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
1428         int progress;
1429         u64 memswlimit;
1430         int ret = 0;
1431
1432         while (retry_count) {
1433                 if (signal_pending(current)) {
1434                         ret = -EINTR;
1435                         break;
1436                 }
1437                 /*
1438                  * Rather than hide all in some function, I do this in
1439                  * open coded manner. You see what this really does.
1440                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1441                  */
1442                 mutex_lock(&set_limit_mutex);
1443                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1444                 if (memswlimit < val) {
1445                         ret = -EINVAL;
1446                         mutex_unlock(&set_limit_mutex);
1447                         break;
1448                 }
1449                 ret = res_counter_set_limit(&memcg->res, val);
1450                 mutex_unlock(&set_limit_mutex);
1451
1452                 if (!ret)
1453                         break;
1454
1455                 progress = try_to_free_mem_cgroup_pages(memcg,
1456                                 GFP_KERNEL, false);
1457                 if (!progress)                  retry_count--;
1458         }
1459
1460         if (!ret)
1461                 mem_cgroup_set_inactive_ratio(memcg);
1462
1463         return ret;
1464 }
1465
1466 int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
1467                                 unsigned long long val)
1468 {
1469         int retry_count = MEM_CGROUP_RECLAIM_RETRIES;
1470         u64 memlimit, oldusage, curusage;
1471         int ret;
1472
1473         if (!do_swap_account)
1474                 return -EINVAL;
1475
1476         while (retry_count) {
1477                 if (signal_pending(current)) {
1478                         ret = -EINTR;
1479                         break;
1480                 }
1481                 /*
1482                  * Rather than hide all in some function, I do this in
1483                  * open coded manner. You see what this really does.
1484                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1485                  */
1486                 mutex_lock(&set_limit_mutex);
1487                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1488                 if (memlimit > val) {
1489                         ret = -EINVAL;
1490                         mutex_unlock(&set_limit_mutex);
1491                         break;
1492                 }
1493                 ret = res_counter_set_limit(&memcg->memsw, val);
1494                 mutex_unlock(&set_limit_mutex);
1495
1496                 if (!ret)
1497                         break;
1498
1499                 oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1500                 try_to_free_mem_cgroup_pages(memcg, GFP_KERNEL, true);
1501                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1502                 if (curusage >= oldusage)
1503                         retry_count--;
1504         }
1505         return ret;
1506 }
1507
1508 /*
1509  * This routine traverse page_cgroup in given list and drop them all.
1510  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
1511  */
1512 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
1513                                 int node, int zid, enum lru_list lru)
1514 {
1515         struct zone *zone;
1516         struct mem_cgroup_per_zone *mz;
1517         struct page_cgroup *pc, *busy;
1518         unsigned long flags, loop;
1519         struct list_head *list;
1520         int ret = 0;
1521
1522         zone = &NODE_DATA(node)->node_zones[zid];
1523         mz = mem_cgroup_zoneinfo(mem, node, zid);
1524         list = &mz->lists[lru];
1525
1526         loop = MEM_CGROUP_ZSTAT(mz, lru);
1527         /* give some margin against EBUSY etc...*/
1528         loop += 256;
1529         busy = NULL;
1530         while (loop--) {
1531                 ret = 0;
1532                 spin_lock_irqsave(&zone->lru_lock, flags);
1533                 if (list_empty(list)) {
1534                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1535                         break;
1536                 }
1537                 pc = list_entry(list->prev, struct page_cgroup, lru);
1538                 if (busy == pc) {
1539                         list_move(&pc->lru, list);
1540                         busy = 0;
1541                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1542                         continue;
1543                 }
1544                 spin_unlock_irqrestore(&zone->lru_lock, flags);
1545
1546                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
1547                 if (ret == -ENOMEM)
1548                         break;
1549
1550                 if (ret == -EBUSY || ret == -EINVAL) {
1551                         /* found lock contention or "pc" is obsolete. */
1552                         busy = pc;
1553                         cond_resched();
1554                 } else
1555                         busy = NULL;
1556         }
1557
1558         if (!ret && !list_empty(list))
1559                 return -EBUSY;
1560         return ret;
1561 }
1562
1563 /*
1564  * make mem_cgroup's charge to be 0 if there is no task.
1565  * This enables deleting this mem_cgroup.
1566  */
1567 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
1568 {
1569         int ret;
1570         int node, zid, shrink;
1571         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1572         struct cgroup *cgrp = mem->css.cgroup;
1573
1574         css_get(&mem->css);
1575
1576         shrink = 0;
1577         /* should free all ? */
1578         if (free_all)
1579                 goto try_to_free;
1580 move_account:
1581         while (mem->res.usage > 0) {
1582                 ret = -EBUSY;
1583                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
1584                         goto out;
1585                 ret = -EINTR;
1586                 if (signal_pending(current))
1587                         goto out;
1588                 /* This is for making all *used* pages to be on LRU. */
1589                 lru_add_drain_all();
1590                 ret = 0;
1591                 for_each_node_state(node, N_POSSIBLE) {
1592                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
1593                                 enum lru_list l;
1594                                 for_each_lru(l) {
1595                                         ret = mem_cgroup_force_empty_list(mem,
1596                                                         node, zid, l);
1597                                         if (ret)
1598                                                 break;
1599                                 }
1600                         }
1601                         if (ret)
1602                                 break;
1603                 }
1604                 /* it seems parent cgroup doesn't have enough mem */
1605                 if (ret == -ENOMEM)
1606                         goto try_to_free;
1607                 cond_resched();
1608         }
1609         ret = 0;
1610 out:
1611         css_put(&mem->css);
1612         return ret;
1613
1614 try_to_free:
1615         /* returns EBUSY if there is a task or if we come here twice. */
1616         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
1617                 ret = -EBUSY;
1618                 goto out;
1619         }
1620         /* we call try-to-free pages for make this cgroup empty */
1621         lru_add_drain_all();
1622         /* try to free all pages in this cgroup */
1623         shrink = 1;
1624         while (nr_retries && mem->res.usage > 0) {
1625                 int progress;
1626
1627                 if (signal_pending(current)) {
1628                         ret = -EINTR;
1629                         goto out;
1630                 }
1631                 progress = try_to_free_mem_cgroup_pages(mem,
1632                                                   GFP_KERNEL, false);
1633                 if (!progress) {
1634                         nr_retries--;
1635                         /* maybe some writeback is necessary */
1636                         congestion_wait(WRITE, HZ/10);
1637                 }
1638
1639         }
1640         lru_add_drain();
1641         /* try move_account...there may be some *locked* pages. */
1642         if (mem->res.usage)
1643                 goto move_account;
1644         ret = 0;
1645         goto out;
1646 }
1647
1648 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
1649 {
1650         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
1651 }
1652
1653
1654 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
1655 {
1656         return mem_cgroup_from_cont(cont)->use_hierarchy;
1657 }
1658
1659 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
1660                                         u64 val)
1661 {
1662         int retval = 0;
1663         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1664         struct cgroup *parent = cont->parent;
1665         struct mem_cgroup *parent_mem = NULL;
1666
1667         if (parent)
1668                 parent_mem = mem_cgroup_from_cont(parent);
1669
1670         cgroup_lock();
1671         /*
1672          * If parent's use_hiearchy is set, we can't make any modifications
1673          * in the child subtrees. If it is unset, then the change can
1674          * occur, provided the current cgroup has no children.
1675          *
1676          * For the root cgroup, parent_mem is NULL, we allow value to be
1677          * set if there are no children.
1678          */
1679         if ((!parent_mem || !parent_mem->use_hierarchy) &&
1680                                 (val == 1 || val == 0)) {
1681                 if (list_empty(&cont->children))
1682                         mem->use_hierarchy = val;
1683                 else
1684                         retval = -EBUSY;
1685         } else
1686                 retval = -EINVAL;
1687         cgroup_unlock();
1688
1689         return retval;
1690 }
1691
1692 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
1693 {
1694         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1695         u64 val = 0;
1696         int type, name;
1697
1698         type = MEMFILE_TYPE(cft->private);
1699         name = MEMFILE_ATTR(cft->private);
1700         switch (type) {
1701         case _MEM:
1702                 val = res_counter_read_u64(&mem->res, name);
1703                 break;
1704         case _MEMSWAP:
1705                 if (do_swap_account)
1706                         val = res_counter_read_u64(&mem->memsw, name);
1707                 break;
1708         default:
1709                 BUG();
1710                 break;
1711         }
1712         return val;
1713 }
1714 /*
1715  * The user of this function is...
1716  * RES_LIMIT.
1717  */
1718 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
1719                             const char *buffer)
1720 {
1721         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
1722         int type, name;
1723         unsigned long long val;
1724         int ret;
1725
1726         type = MEMFILE_TYPE(cft->private);
1727         name = MEMFILE_ATTR(cft->private);
1728         switch (name) {
1729         case RES_LIMIT:
1730                 /* This function does all necessary parse...reuse it */
1731                 ret = res_counter_memparse_write_strategy(buffer, &val);
1732                 if (ret)
1733                         break;
1734                 if (type == _MEM)
1735                         ret = mem_cgroup_resize_limit(memcg, val);
1736                 else
1737                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
1738                 break;
1739         default:
1740                 ret = -EINVAL; /* should be BUG() ? */
1741                 break;
1742         }
1743         return ret;
1744 }
1745
1746 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
1747 {
1748         struct mem_cgroup *mem;
1749         int type, name;
1750
1751         mem = mem_cgroup_from_cont(cont);
1752         type = MEMFILE_TYPE(event);
1753         name = MEMFILE_ATTR(event);
1754         switch (name) {
1755         case RES_MAX_USAGE:
1756                 if (type == _MEM)
1757                         res_counter_reset_max(&mem->res);
1758                 else
1759                         res_counter_reset_max(&mem->memsw);
1760                 break;
1761         case RES_FAILCNT:
1762                 if (type == _MEM)
1763                         res_counter_reset_failcnt(&mem->res);
1764                 else
1765                         res_counter_reset_failcnt(&mem->memsw);
1766                 break;
1767         }
1768         return 0;
1769 }
1770
1771 static const struct mem_cgroup_stat_desc {
1772         const char *msg;
1773         u64 unit;
1774 } mem_cgroup_stat_desc[] = {
1775         [MEM_CGROUP_STAT_CACHE] = { "cache", PAGE_SIZE, },
1776         [MEM_CGROUP_STAT_RSS] = { "rss", PAGE_SIZE, },
1777         [MEM_CGROUP_STAT_PGPGIN_COUNT] = {"pgpgin", 1, },
1778         [MEM_CGROUP_STAT_PGPGOUT_COUNT] = {"pgpgout", 1, },
1779 };
1780
1781 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
1782                                  struct cgroup_map_cb *cb)
1783 {
1784         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
1785         struct mem_cgroup_stat *stat = &mem_cont->stat;
1786         int i;
1787
1788         for (i = 0; i < ARRAY_SIZE(stat->cpustat[0].count); i++) {
1789                 s64 val;
1790
1791                 val = mem_cgroup_read_stat(stat, i);
1792                 val *= mem_cgroup_stat_desc[i].unit;
1793                 cb->fill(cb, mem_cgroup_stat_desc[i].msg, val);
1794         }
1795         /* showing # of active pages */
1796         {
1797                 unsigned long active_anon, inactive_anon;
1798                 unsigned long active_file, inactive_file;
1799                 unsigned long unevictable;
1800
1801                 inactive_anon = mem_cgroup_get_all_zonestat(mem_cont,
1802                                                 LRU_INACTIVE_ANON);
1803                 active_anon = mem_cgroup_get_all_zonestat(mem_cont,
1804                                                 LRU_ACTIVE_ANON);
1805                 inactive_file = mem_cgroup_get_all_zonestat(mem_cont,
1806                                                 LRU_INACTIVE_FILE);
1807                 active_file = mem_cgroup_get_all_zonestat(mem_cont,
1808                                                 LRU_ACTIVE_FILE);
1809                 unevictable = mem_cgroup_get_all_zonestat(mem_cont,
1810                                                         LRU_UNEVICTABLE);
1811
1812                 cb->fill(cb, "active_anon", (active_anon) * PAGE_SIZE);
1813                 cb->fill(cb, "inactive_anon", (inactive_anon) * PAGE_SIZE);
1814                 cb->fill(cb, "active_file", (active_file) * PAGE_SIZE);
1815                 cb->fill(cb, "inactive_file", (inactive_file) * PAGE_SIZE);
1816                 cb->fill(cb, "unevictable", unevictable * PAGE_SIZE);
1817
1818         }
1819         return 0;
1820 }
1821
1822
1823 static struct cftype mem_cgroup_files[] = {
1824         {
1825                 .name = "usage_in_bytes",
1826                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
1827                 .read_u64 = mem_cgroup_read,
1828         },
1829         {
1830                 .name = "max_usage_in_bytes",
1831                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
1832                 .trigger = mem_cgroup_reset,
1833                 .read_u64 = mem_cgroup_read,
1834         },
1835         {
1836                 .name = "limit_in_bytes",
1837                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
1838                 .write_string = mem_cgroup_write,
1839                 .read_u64 = mem_cgroup_read,
1840         },
1841         {
1842                 .name = "failcnt",
1843                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
1844                 .trigger = mem_cgroup_reset,
1845                 .read_u64 = mem_cgroup_read,
1846         },
1847         {
1848                 .name = "stat",
1849                 .read_map = mem_control_stat_show,
1850         },
1851         {
1852                 .name = "force_empty",
1853                 .trigger = mem_cgroup_force_empty_write,
1854         },
1855         {
1856                 .name = "use_hierarchy",
1857                 .write_u64 = mem_cgroup_hierarchy_write,
1858                 .read_u64 = mem_cgroup_hierarchy_read,
1859         },
1860 };
1861
1862 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1863 static struct cftype memsw_cgroup_files[] = {
1864         {
1865                 .name = "memsw.usage_in_bytes",
1866                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
1867                 .read_u64 = mem_cgroup_read,
1868         },
1869         {
1870                 .name = "memsw.max_usage_in_bytes",
1871                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
1872                 .trigger = mem_cgroup_reset,
1873                 .read_u64 = mem_cgroup_read,
1874         },
1875         {
1876                 .name = "memsw.limit_in_bytes",
1877                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
1878                 .write_string = mem_cgroup_write,
1879                 .read_u64 = mem_cgroup_read,
1880         },
1881         {
1882                 .name = "memsw.failcnt",
1883                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
1884                 .trigger = mem_cgroup_reset,
1885                 .read_u64 = mem_cgroup_read,
1886         },
1887 };
1888
1889 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
1890 {
1891         if (!do_swap_account)
1892                 return 0;
1893         return cgroup_add_files(cont, ss, memsw_cgroup_files,
1894                                 ARRAY_SIZE(memsw_cgroup_files));
1895 };
1896 #else
1897 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
1898 {
1899         return 0;
1900 }
1901 #endif
1902
1903 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1904 {
1905         struct mem_cgroup_per_node *pn;
1906         struct mem_cgroup_per_zone *mz;
1907         enum lru_list l;
1908         int zone, tmp = node;
1909         /*
1910          * This routine is called against possible nodes.
1911          * But it's BUG to call kmalloc() against offline node.
1912          *
1913          * TODO: this routine can waste much memory for nodes which will
1914          *       never be onlined. It's better to use memory hotplug callback
1915          *       function.
1916          */
1917         if (!node_state(node, N_NORMAL_MEMORY))
1918                 tmp = -1;
1919         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
1920         if (!pn)
1921                 return 1;
1922
1923         mem->info.nodeinfo[node] = pn;
1924         memset(pn, 0, sizeof(*pn));
1925
1926         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
1927                 mz = &pn->zoneinfo[zone];
1928                 for_each_lru(l)
1929                         INIT_LIST_HEAD(&mz->lists[l]);
1930         }
1931         return 0;
1932 }
1933
1934 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
1935 {
1936         kfree(mem->info.nodeinfo[node]);
1937 }
1938
1939 static int mem_cgroup_size(void)
1940 {
1941         int cpustat_size = nr_cpu_ids * sizeof(struct mem_cgroup_stat_cpu);
1942         return sizeof(struct mem_cgroup) + cpustat_size;
1943 }
1944
1945 static struct mem_cgroup *mem_cgroup_alloc(void)
1946 {
1947         struct mem_cgroup *mem;
1948         int size = mem_cgroup_size();
1949
1950         if (size < PAGE_SIZE)
1951                 mem = kmalloc(size, GFP_KERNEL);
1952         else
1953                 mem = vmalloc(size);
1954
1955         if (mem)
1956                 memset(mem, 0, size);
1957         return mem;
1958 }
1959
1960 /*
1961  * At destroying mem_cgroup, references from swap_cgroup can remain.
1962  * (scanning all at force_empty is too costly...)
1963  *
1964  * Instead of clearing all references at force_empty, we remember
1965  * the number of reference from swap_cgroup and free mem_cgroup when
1966  * it goes down to 0.
1967  *
1968  * When mem_cgroup is destroyed, mem->obsolete will be set to 0 and
1969  * entry which points to this memcg will be ignore at swapin.
1970  *
1971  * Removal of cgroup itself succeeds regardless of refs from swap.
1972  */
1973
1974 static void mem_cgroup_free(struct mem_cgroup *mem)
1975 {
1976         int node;
1977
1978         if (atomic_read(&mem->refcnt) > 0)
1979                 return;
1980
1981
1982         for_each_node_state(node, N_POSSIBLE)
1983                 free_mem_cgroup_per_zone_info(mem, node);
1984
1985         if (mem_cgroup_size() < PAGE_SIZE)
1986                 kfree(mem);
1987         else
1988                 vfree(mem);
1989 }
1990
1991 static void mem_cgroup_get(struct mem_cgroup *mem)
1992 {
1993         atomic_inc(&mem->refcnt);
1994 }
1995
1996 static void mem_cgroup_put(struct mem_cgroup *mem)
1997 {
1998         if (atomic_dec_and_test(&mem->refcnt)) {
1999                 if (!mem->obsolete)
2000                         return;
2001                 mem_cgroup_free(mem);
2002         }
2003 }
2004
2005
2006 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2007 static void __init enable_swap_cgroup(void)
2008 {
2009         if (!mem_cgroup_disabled() && really_do_swap_account)
2010                 do_swap_account = 1;
2011 }
2012 #else
2013 static void __init enable_swap_cgroup(void)
2014 {
2015 }
2016 #endif
2017
2018 static struct cgroup_subsys_state *
2019 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
2020 {
2021         struct mem_cgroup *mem, *parent;
2022         int node;
2023
2024         mem = mem_cgroup_alloc();
2025         if (!mem)
2026                 return ERR_PTR(-ENOMEM);
2027
2028         for_each_node_state(node, N_POSSIBLE)
2029                 if (alloc_mem_cgroup_per_zone_info(mem, node))
2030                         goto free_out;
2031         /* root ? */
2032         if (cont->parent == NULL) {
2033                 enable_swap_cgroup();
2034                 parent = NULL;
2035         } else {
2036                 parent = mem_cgroup_from_cont(cont->parent);
2037                 mem->use_hierarchy = parent->use_hierarchy;
2038         }
2039
2040         if (parent && parent->use_hierarchy) {
2041                 res_counter_init(&mem->res, &parent->res);
2042                 res_counter_init(&mem->memsw, &parent->memsw);
2043         } else {
2044                 res_counter_init(&mem->res, NULL);
2045                 res_counter_init(&mem->memsw, NULL);
2046         }
2047         mem_cgroup_set_inactive_ratio(mem);
2048         mem->last_scanned_child = NULL;
2049
2050         return &mem->css;
2051 free_out:
2052         for_each_node_state(node, N_POSSIBLE)
2053                 free_mem_cgroup_per_zone_info(mem, node);
2054         mem_cgroup_free(mem);
2055         return ERR_PTR(-ENOMEM);
2056 }
2057
2058 static void mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
2059                                         struct cgroup *cont)
2060 {
2061         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
2062         mem->obsolete = 1;
2063         mem_cgroup_force_empty(mem, false);
2064 }
2065
2066 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
2067                                 struct cgroup *cont)
2068 {
2069         mem_cgroup_free(mem_cgroup_from_cont(cont));
2070 }
2071
2072 static int mem_cgroup_populate(struct cgroup_subsys *ss,
2073                                 struct cgroup *cont)
2074 {
2075         int ret;
2076
2077         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
2078                                 ARRAY_SIZE(mem_cgroup_files));
2079
2080         if (!ret)
2081                 ret = register_memsw_files(cont, ss);
2082         return ret;
2083 }
2084
2085 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
2086                                 struct cgroup *cont,
2087                                 struct cgroup *old_cont,
2088                                 struct task_struct *p)
2089 {
2090         /*
2091          * FIXME: It's better to move charges of this process from old
2092          * memcg to new memcg. But it's just on TODO-List now.
2093          */
2094 }
2095
2096 struct cgroup_subsys mem_cgroup_subsys = {
2097         .name = "memory",
2098         .subsys_id = mem_cgroup_subsys_id,
2099         .create = mem_cgroup_create,
2100         .pre_destroy = mem_cgroup_pre_destroy,
2101         .destroy = mem_cgroup_destroy,
2102         .populate = mem_cgroup_populate,
2103         .attach = mem_cgroup_move_task,
2104         .early_init = 0,
2105 };
2106
2107 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2108
2109 static int __init disable_swap_account(char *s)
2110 {
2111         really_do_swap_account = 0;
2112         return 1;
2113 }
2114 __setup("noswapaccount", disable_swap_account);
2115 #endif