page-allocator: reset wmark_min and inactive ratio of zone when hotplug happens
[linux-2.6.git] / mm / memcontrol.c
1 /* memcontrol.c - Memory Controller
2  *
3  * Copyright IBM Corporation, 2007
4  * Author Balbir Singh <balbir@linux.vnet.ibm.com>
5  *
6  * Copyright 2007 OpenVZ SWsoft Inc
7  * Author: Pavel Emelianov <xemul@openvz.org>
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License as published by
11  * the Free Software Foundation; either version 2 of the License, or
12  * (at your option) any later version.
13  *
14  * This program is distributed in the hope that it will be useful,
15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17  * GNU General Public License for more details.
18  */
19
20 #include <linux/res_counter.h>
21 #include <linux/memcontrol.h>
22 #include <linux/cgroup.h>
23 #include <linux/mm.h>
24 #include <linux/pagemap.h>
25 #include <linux/smp.h>
26 #include <linux/page-flags.h>
27 #include <linux/backing-dev.h>
28 #include <linux/bit_spinlock.h>
29 #include <linux/rcupdate.h>
30 #include <linux/limits.h>
31 #include <linux/mutex.h>
32 #include <linux/slab.h>
33 #include <linux/swap.h>
34 #include <linux/spinlock.h>
35 #include <linux/fs.h>
36 #include <linux/seq_file.h>
37 #include <linux/vmalloc.h>
38 #include <linux/mm_inline.h>
39 #include <linux/page_cgroup.h>
40 #include "internal.h"
41
42 #include <asm/uaccess.h>
43
44 struct cgroup_subsys mem_cgroup_subsys __read_mostly;
45 #define MEM_CGROUP_RECLAIM_RETRIES      5
46
47 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
48 /* Turned on only when memory cgroup is enabled && really_do_swap_account = 0 */
49 int do_swap_account __read_mostly;
50 static int really_do_swap_account __initdata = 1; /* for remember boot option*/
51 #else
52 #define do_swap_account         (0)
53 #endif
54
55 static DEFINE_MUTEX(memcg_tasklist);    /* can be hold under cgroup_mutex */
56
57 /*
58  * Statistics for memory cgroup.
59  */
60 enum mem_cgroup_stat_index {
61         /*
62          * For MEM_CONTAINER_TYPE_ALL, usage = pagecache + rss.
63          */
64         MEM_CGROUP_STAT_CACHE,     /* # of pages charged as cache */
65         MEM_CGROUP_STAT_RSS,       /* # of pages charged as rss */
66         MEM_CGROUP_STAT_PGPGIN_COUNT,   /* # of pages paged in */
67         MEM_CGROUP_STAT_PGPGOUT_COUNT,  /* # of pages paged out */
68
69         MEM_CGROUP_STAT_NSTATS,
70 };
71
72 struct mem_cgroup_stat_cpu {
73         s64 count[MEM_CGROUP_STAT_NSTATS];
74 } ____cacheline_aligned_in_smp;
75
76 struct mem_cgroup_stat {
77         struct mem_cgroup_stat_cpu cpustat[0];
78 };
79
80 /*
81  * For accounting under irq disable, no need for increment preempt count.
82  */
83 static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
84                 enum mem_cgroup_stat_index idx, int val)
85 {
86         stat->count[idx] += val;
87 }
88
89 static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
90                 enum mem_cgroup_stat_index idx)
91 {
92         int cpu;
93         s64 ret = 0;
94         for_each_possible_cpu(cpu)
95                 ret += stat->cpustat[cpu].count[idx];
96         return ret;
97 }
98
99 static s64 mem_cgroup_local_usage(struct mem_cgroup_stat *stat)
100 {
101         s64 ret;
102
103         ret = mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_CACHE);
104         ret += mem_cgroup_read_stat(stat, MEM_CGROUP_STAT_RSS);
105         return ret;
106 }
107
108 /*
109  * per-zone information in memory controller.
110  */
111 struct mem_cgroup_per_zone {
112         /*
113          * spin_lock to protect the per cgroup LRU
114          */
115         struct list_head        lists[NR_LRU_LISTS];
116         unsigned long           count[NR_LRU_LISTS];
117
118         struct zone_reclaim_stat reclaim_stat;
119 };
120 /* Macro for accessing counter */
121 #define MEM_CGROUP_ZSTAT(mz, idx)       ((mz)->count[(idx)])
122
123 struct mem_cgroup_per_node {
124         struct mem_cgroup_per_zone zoneinfo[MAX_NR_ZONES];
125 };
126
127 struct mem_cgroup_lru_info {
128         struct mem_cgroup_per_node *nodeinfo[MAX_NUMNODES];
129 };
130
131 /*
132  * The memory controller data structure. The memory controller controls both
133  * page cache and RSS per cgroup. We would eventually like to provide
134  * statistics based on the statistics developed by Rik Van Riel for clock-pro,
135  * to help the administrator determine what knobs to tune.
136  *
137  * TODO: Add a water mark for the memory controller. Reclaim will begin when
138  * we hit the water mark. May be even add a low water mark, such that
139  * no reclaim occurs from a cgroup at it's low water mark, this is
140  * a feature that will be implemented much later in the future.
141  */
142 struct mem_cgroup {
143         struct cgroup_subsys_state css;
144         /*
145          * the counter to account for memory usage
146          */
147         struct res_counter res;
148         /*
149          * the counter to account for mem+swap usage.
150          */
151         struct res_counter memsw;
152         /*
153          * Per cgroup active and inactive list, similar to the
154          * per zone LRU lists.
155          */
156         struct mem_cgroup_lru_info info;
157
158         /*
159           protect against reclaim related member.
160         */
161         spinlock_t reclaim_param_lock;
162
163         int     prev_priority;  /* for recording reclaim priority */
164
165         /*
166          * While reclaiming in a hiearchy, we cache the last child we
167          * reclaimed from.
168          */
169         int last_scanned_child;
170         /*
171          * Should the accounting and control be hierarchical, per subtree?
172          */
173         bool use_hierarchy;
174         unsigned long   last_oom_jiffies;
175         atomic_t        refcnt;
176
177         unsigned int    swappiness;
178
179         /*
180          * statistics. This must be placed at the end of memcg.
181          */
182         struct mem_cgroup_stat stat;
183 };
184
185 enum charge_type {
186         MEM_CGROUP_CHARGE_TYPE_CACHE = 0,
187         MEM_CGROUP_CHARGE_TYPE_MAPPED,
188         MEM_CGROUP_CHARGE_TYPE_SHMEM,   /* used by page migration of shmem */
189         MEM_CGROUP_CHARGE_TYPE_FORCE,   /* used by force_empty */
190         MEM_CGROUP_CHARGE_TYPE_SWAPOUT, /* for accounting swapcache */
191         NR_CHARGE_TYPE,
192 };
193
194 /* only for here (for easy reading.) */
195 #define PCGF_CACHE      (1UL << PCG_CACHE)
196 #define PCGF_USED       (1UL << PCG_USED)
197 #define PCGF_LOCK       (1UL << PCG_LOCK)
198 static const unsigned long
199 pcg_default_flags[NR_CHARGE_TYPE] = {
200         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* File Cache */
201         PCGF_USED | PCGF_LOCK, /* Anon */
202         PCGF_CACHE | PCGF_USED | PCGF_LOCK, /* Shmem */
203         0, /* FORCE */
204 };
205
206 /* for encoding cft->private value on file */
207 #define _MEM                    (0)
208 #define _MEMSWAP                (1)
209 #define MEMFILE_PRIVATE(x, val) (((x) << 16) | (val))
210 #define MEMFILE_TYPE(val)       (((val) >> 16) & 0xffff)
211 #define MEMFILE_ATTR(val)       ((val) & 0xffff)
212
213 static void mem_cgroup_get(struct mem_cgroup *mem);
214 static void mem_cgroup_put(struct mem_cgroup *mem);
215 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem);
216
217 static void mem_cgroup_charge_statistics(struct mem_cgroup *mem,
218                                          struct page_cgroup *pc,
219                                          bool charge)
220 {
221         int val = (charge)? 1 : -1;
222         struct mem_cgroup_stat *stat = &mem->stat;
223         struct mem_cgroup_stat_cpu *cpustat;
224         int cpu = get_cpu();
225
226         cpustat = &stat->cpustat[cpu];
227         if (PageCgroupCache(pc))
228                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
229         else
230                 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
231
232         if (charge)
233                 __mem_cgroup_stat_add_safe(cpustat,
234                                 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
235         else
236                 __mem_cgroup_stat_add_safe(cpustat,
237                                 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
238         put_cpu();
239 }
240
241 static struct mem_cgroup_per_zone *
242 mem_cgroup_zoneinfo(struct mem_cgroup *mem, int nid, int zid)
243 {
244         return &mem->info.nodeinfo[nid]->zoneinfo[zid];
245 }
246
247 static struct mem_cgroup_per_zone *
248 page_cgroup_zoneinfo(struct page_cgroup *pc)
249 {
250         struct mem_cgroup *mem = pc->mem_cgroup;
251         int nid = page_cgroup_nid(pc);
252         int zid = page_cgroup_zid(pc);
253
254         if (!mem)
255                 return NULL;
256
257         return mem_cgroup_zoneinfo(mem, nid, zid);
258 }
259
260 static unsigned long mem_cgroup_get_local_zonestat(struct mem_cgroup *mem,
261                                         enum lru_list idx)
262 {
263         int nid, zid;
264         struct mem_cgroup_per_zone *mz;
265         u64 total = 0;
266
267         for_each_online_node(nid)
268                 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
269                         mz = mem_cgroup_zoneinfo(mem, nid, zid);
270                         total += MEM_CGROUP_ZSTAT(mz, idx);
271                 }
272         return total;
273 }
274
275 static struct mem_cgroup *mem_cgroup_from_cont(struct cgroup *cont)
276 {
277         return container_of(cgroup_subsys_state(cont,
278                                 mem_cgroup_subsys_id), struct mem_cgroup,
279                                 css);
280 }
281
282 struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p)
283 {
284         /*
285          * mm_update_next_owner() may clear mm->owner to NULL
286          * if it races with swapoff, page migration, etc.
287          * So this can be called with p == NULL.
288          */
289         if (unlikely(!p))
290                 return NULL;
291
292         return container_of(task_subsys_state(p, mem_cgroup_subsys_id),
293                                 struct mem_cgroup, css);
294 }
295
296 static struct mem_cgroup *try_get_mem_cgroup_from_mm(struct mm_struct *mm)
297 {
298         struct mem_cgroup *mem = NULL;
299
300         if (!mm)
301                 return NULL;
302         /*
303          * Because we have no locks, mm->owner's may be being moved to other
304          * cgroup. We use css_tryget() here even if this looks
305          * pessimistic (rather than adding locks here).
306          */
307         rcu_read_lock();
308         do {
309                 mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
310                 if (unlikely(!mem))
311                         break;
312         } while (!css_tryget(&mem->css));
313         rcu_read_unlock();
314         return mem;
315 }
316
317 /*
318  * Call callback function against all cgroup under hierarchy tree.
319  */
320 static int mem_cgroup_walk_tree(struct mem_cgroup *root, void *data,
321                           int (*func)(struct mem_cgroup *, void *))
322 {
323         int found, ret, nextid;
324         struct cgroup_subsys_state *css;
325         struct mem_cgroup *mem;
326
327         if (!root->use_hierarchy)
328                 return (*func)(root, data);
329
330         nextid = 1;
331         do {
332                 ret = 0;
333                 mem = NULL;
334
335                 rcu_read_lock();
336                 css = css_get_next(&mem_cgroup_subsys, nextid, &root->css,
337                                    &found);
338                 if (css && css_tryget(css))
339                         mem = container_of(css, struct mem_cgroup, css);
340                 rcu_read_unlock();
341
342                 if (mem) {
343                         ret = (*func)(mem, data);
344                         css_put(&mem->css);
345                 }
346                 nextid = found + 1;
347         } while (!ret && css);
348
349         return ret;
350 }
351
352 /*
353  * Following LRU functions are allowed to be used without PCG_LOCK.
354  * Operations are called by routine of global LRU independently from memcg.
355  * What we have to take care of here is validness of pc->mem_cgroup.
356  *
357  * Changes to pc->mem_cgroup happens when
358  * 1. charge
359  * 2. moving account
360  * In typical case, "charge" is done before add-to-lru. Exception is SwapCache.
361  * It is added to LRU before charge.
362  * If PCG_USED bit is not set, page_cgroup is not added to this private LRU.
363  * When moving account, the page is not on LRU. It's isolated.
364  */
365
366 void mem_cgroup_del_lru_list(struct page *page, enum lru_list lru)
367 {
368         struct page_cgroup *pc;
369         struct mem_cgroup *mem;
370         struct mem_cgroup_per_zone *mz;
371
372         if (mem_cgroup_disabled())
373                 return;
374         pc = lookup_page_cgroup(page);
375         /* can happen while we handle swapcache. */
376         if (list_empty(&pc->lru) || !pc->mem_cgroup)
377                 return;
378         /*
379          * We don't check PCG_USED bit. It's cleared when the "page" is finally
380          * removed from global LRU.
381          */
382         mz = page_cgroup_zoneinfo(pc);
383         mem = pc->mem_cgroup;
384         MEM_CGROUP_ZSTAT(mz, lru) -= 1;
385         list_del_init(&pc->lru);
386         return;
387 }
388
389 void mem_cgroup_del_lru(struct page *page)
390 {
391         mem_cgroup_del_lru_list(page, page_lru(page));
392 }
393
394 void mem_cgroup_rotate_lru_list(struct page *page, enum lru_list lru)
395 {
396         struct mem_cgroup_per_zone *mz;
397         struct page_cgroup *pc;
398
399         if (mem_cgroup_disabled())
400                 return;
401
402         pc = lookup_page_cgroup(page);
403         /*
404          * Used bit is set without atomic ops but after smp_wmb().
405          * For making pc->mem_cgroup visible, insert smp_rmb() here.
406          */
407         smp_rmb();
408         /* unused page is not rotated. */
409         if (!PageCgroupUsed(pc))
410                 return;
411         mz = page_cgroup_zoneinfo(pc);
412         list_move(&pc->lru, &mz->lists[lru]);
413 }
414
415 void mem_cgroup_add_lru_list(struct page *page, enum lru_list lru)
416 {
417         struct page_cgroup *pc;
418         struct mem_cgroup_per_zone *mz;
419
420         if (mem_cgroup_disabled())
421                 return;
422         pc = lookup_page_cgroup(page);
423         /*
424          * Used bit is set without atomic ops but after smp_wmb().
425          * For making pc->mem_cgroup visible, insert smp_rmb() here.
426          */
427         smp_rmb();
428         if (!PageCgroupUsed(pc))
429                 return;
430
431         mz = page_cgroup_zoneinfo(pc);
432         MEM_CGROUP_ZSTAT(mz, lru) += 1;
433         list_add(&pc->lru, &mz->lists[lru]);
434 }
435
436 /*
437  * At handling SwapCache, pc->mem_cgroup may be changed while it's linked to
438  * lru because the page may.be reused after it's fully uncharged (because of
439  * SwapCache behavior).To handle that, unlink page_cgroup from LRU when charge
440  * it again. This function is only used to charge SwapCache. It's done under
441  * lock_page and expected that zone->lru_lock is never held.
442  */
443 static void mem_cgroup_lru_del_before_commit_swapcache(struct page *page)
444 {
445         unsigned long flags;
446         struct zone *zone = page_zone(page);
447         struct page_cgroup *pc = lookup_page_cgroup(page);
448
449         spin_lock_irqsave(&zone->lru_lock, flags);
450         /*
451          * Forget old LRU when this page_cgroup is *not* used. This Used bit
452          * is guarded by lock_page() because the page is SwapCache.
453          */
454         if (!PageCgroupUsed(pc))
455                 mem_cgroup_del_lru_list(page, page_lru(page));
456         spin_unlock_irqrestore(&zone->lru_lock, flags);
457 }
458
459 static void mem_cgroup_lru_add_after_commit_swapcache(struct page *page)
460 {
461         unsigned long flags;
462         struct zone *zone = page_zone(page);
463         struct page_cgroup *pc = lookup_page_cgroup(page);
464
465         spin_lock_irqsave(&zone->lru_lock, flags);
466         /* link when the page is linked to LRU but page_cgroup isn't */
467         if (PageLRU(page) && list_empty(&pc->lru))
468                 mem_cgroup_add_lru_list(page, page_lru(page));
469         spin_unlock_irqrestore(&zone->lru_lock, flags);
470 }
471
472
473 void mem_cgroup_move_lists(struct page *page,
474                            enum lru_list from, enum lru_list to)
475 {
476         if (mem_cgroup_disabled())
477                 return;
478         mem_cgroup_del_lru_list(page, from);
479         mem_cgroup_add_lru_list(page, to);
480 }
481
482 int task_in_mem_cgroup(struct task_struct *task, const struct mem_cgroup *mem)
483 {
484         int ret;
485         struct mem_cgroup *curr = NULL;
486
487         task_lock(task);
488         rcu_read_lock();
489         curr = try_get_mem_cgroup_from_mm(task->mm);
490         rcu_read_unlock();
491         task_unlock(task);
492         if (!curr)
493                 return 0;
494         if (curr->use_hierarchy)
495                 ret = css_is_ancestor(&curr->css, &mem->css);
496         else
497                 ret = (curr == mem);
498         css_put(&curr->css);
499         return ret;
500 }
501
502 /*
503  * prev_priority control...this will be used in memory reclaim path.
504  */
505 int mem_cgroup_get_reclaim_priority(struct mem_cgroup *mem)
506 {
507         int prev_priority;
508
509         spin_lock(&mem->reclaim_param_lock);
510         prev_priority = mem->prev_priority;
511         spin_unlock(&mem->reclaim_param_lock);
512
513         return prev_priority;
514 }
515
516 void mem_cgroup_note_reclaim_priority(struct mem_cgroup *mem, int priority)
517 {
518         spin_lock(&mem->reclaim_param_lock);
519         if (priority < mem->prev_priority)
520                 mem->prev_priority = priority;
521         spin_unlock(&mem->reclaim_param_lock);
522 }
523
524 void mem_cgroup_record_reclaim_priority(struct mem_cgroup *mem, int priority)
525 {
526         spin_lock(&mem->reclaim_param_lock);
527         mem->prev_priority = priority;
528         spin_unlock(&mem->reclaim_param_lock);
529 }
530
531 static int calc_inactive_ratio(struct mem_cgroup *memcg, unsigned long *present_pages)
532 {
533         unsigned long active;
534         unsigned long inactive;
535         unsigned long gb;
536         unsigned long inactive_ratio;
537
538         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_ANON);
539         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_ANON);
540
541         gb = (inactive + active) >> (30 - PAGE_SHIFT);
542         if (gb)
543                 inactive_ratio = int_sqrt(10 * gb);
544         else
545                 inactive_ratio = 1;
546
547         if (present_pages) {
548                 present_pages[0] = inactive;
549                 present_pages[1] = active;
550         }
551
552         return inactive_ratio;
553 }
554
555 int mem_cgroup_inactive_anon_is_low(struct mem_cgroup *memcg)
556 {
557         unsigned long active;
558         unsigned long inactive;
559         unsigned long present_pages[2];
560         unsigned long inactive_ratio;
561
562         inactive_ratio = calc_inactive_ratio(memcg, present_pages);
563
564         inactive = present_pages[0];
565         active = present_pages[1];
566
567         if (inactive * inactive_ratio < active)
568                 return 1;
569
570         return 0;
571 }
572
573 int mem_cgroup_inactive_file_is_low(struct mem_cgroup *memcg)
574 {
575         unsigned long active;
576         unsigned long inactive;
577
578         inactive = mem_cgroup_get_local_zonestat(memcg, LRU_INACTIVE_FILE);
579         active = mem_cgroup_get_local_zonestat(memcg, LRU_ACTIVE_FILE);
580
581         return (active > inactive);
582 }
583
584 unsigned long mem_cgroup_zone_nr_pages(struct mem_cgroup *memcg,
585                                        struct zone *zone,
586                                        enum lru_list lru)
587 {
588         int nid = zone->zone_pgdat->node_id;
589         int zid = zone_idx(zone);
590         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
591
592         return MEM_CGROUP_ZSTAT(mz, lru);
593 }
594
595 struct zone_reclaim_stat *mem_cgroup_get_reclaim_stat(struct mem_cgroup *memcg,
596                                                       struct zone *zone)
597 {
598         int nid = zone->zone_pgdat->node_id;
599         int zid = zone_idx(zone);
600         struct mem_cgroup_per_zone *mz = mem_cgroup_zoneinfo(memcg, nid, zid);
601
602         return &mz->reclaim_stat;
603 }
604
605 struct zone_reclaim_stat *
606 mem_cgroup_get_reclaim_stat_from_page(struct page *page)
607 {
608         struct page_cgroup *pc;
609         struct mem_cgroup_per_zone *mz;
610
611         if (mem_cgroup_disabled())
612                 return NULL;
613
614         pc = lookup_page_cgroup(page);
615         /*
616          * Used bit is set without atomic ops but after smp_wmb().
617          * For making pc->mem_cgroup visible, insert smp_rmb() here.
618          */
619         smp_rmb();
620         if (!PageCgroupUsed(pc))
621                 return NULL;
622
623         mz = page_cgroup_zoneinfo(pc);
624         if (!mz)
625                 return NULL;
626
627         return &mz->reclaim_stat;
628 }
629
630 unsigned long mem_cgroup_isolate_pages(unsigned long nr_to_scan,
631                                         struct list_head *dst,
632                                         unsigned long *scanned, int order,
633                                         int mode, struct zone *z,
634                                         struct mem_cgroup *mem_cont,
635                                         int active, int file)
636 {
637         unsigned long nr_taken = 0;
638         struct page *page;
639         unsigned long scan;
640         LIST_HEAD(pc_list);
641         struct list_head *src;
642         struct page_cgroup *pc, *tmp;
643         int nid = z->zone_pgdat->node_id;
644         int zid = zone_idx(z);
645         struct mem_cgroup_per_zone *mz;
646         int lru = LRU_FILE * !!file + !!active;
647
648         BUG_ON(!mem_cont);
649         mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
650         src = &mz->lists[lru];
651
652         scan = 0;
653         list_for_each_entry_safe_reverse(pc, tmp, src, lru) {
654                 if (scan >= nr_to_scan)
655                         break;
656
657                 page = pc->page;
658                 if (unlikely(!PageCgroupUsed(pc)))
659                         continue;
660                 if (unlikely(!PageLRU(page)))
661                         continue;
662
663                 scan++;
664                 if (__isolate_lru_page(page, mode, file) == 0) {
665                         list_move(&page->lru, dst);
666                         nr_taken++;
667                 }
668         }
669
670         *scanned = scan;
671         return nr_taken;
672 }
673
674 #define mem_cgroup_from_res_counter(counter, member)    \
675         container_of(counter, struct mem_cgroup, member)
676
677 static bool mem_cgroup_check_under_limit(struct mem_cgroup *mem)
678 {
679         if (do_swap_account) {
680                 if (res_counter_check_under_limit(&mem->res) &&
681                         res_counter_check_under_limit(&mem->memsw))
682                         return true;
683         } else
684                 if (res_counter_check_under_limit(&mem->res))
685                         return true;
686         return false;
687 }
688
689 static unsigned int get_swappiness(struct mem_cgroup *memcg)
690 {
691         struct cgroup *cgrp = memcg->css.cgroup;
692         unsigned int swappiness;
693
694         /* root ? */
695         if (cgrp->parent == NULL)
696                 return vm_swappiness;
697
698         spin_lock(&memcg->reclaim_param_lock);
699         swappiness = memcg->swappiness;
700         spin_unlock(&memcg->reclaim_param_lock);
701
702         return swappiness;
703 }
704
705 static int mem_cgroup_count_children_cb(struct mem_cgroup *mem, void *data)
706 {
707         int *val = data;
708         (*val)++;
709         return 0;
710 }
711
712 /**
713  * mem_cgroup_print_mem_info: Called from OOM with tasklist_lock held in read mode.
714  * @memcg: The memory cgroup that went over limit
715  * @p: Task that is going to be killed
716  *
717  * NOTE: @memcg and @p's mem_cgroup can be different when hierarchy is
718  * enabled
719  */
720 void mem_cgroup_print_oom_info(struct mem_cgroup *memcg, struct task_struct *p)
721 {
722         struct cgroup *task_cgrp;
723         struct cgroup *mem_cgrp;
724         /*
725          * Need a buffer in BSS, can't rely on allocations. The code relies
726          * on the assumption that OOM is serialized for memory controller.
727          * If this assumption is broken, revisit this code.
728          */
729         static char memcg_name[PATH_MAX];
730         int ret;
731
732         if (!memcg)
733                 return;
734
735
736         rcu_read_lock();
737
738         mem_cgrp = memcg->css.cgroup;
739         task_cgrp = task_cgroup(p, mem_cgroup_subsys_id);
740
741         ret = cgroup_path(task_cgrp, memcg_name, PATH_MAX);
742         if (ret < 0) {
743                 /*
744                  * Unfortunately, we are unable to convert to a useful name
745                  * But we'll still print out the usage information
746                  */
747                 rcu_read_unlock();
748                 goto done;
749         }
750         rcu_read_unlock();
751
752         printk(KERN_INFO "Task in %s killed", memcg_name);
753
754         rcu_read_lock();
755         ret = cgroup_path(mem_cgrp, memcg_name, PATH_MAX);
756         if (ret < 0) {
757                 rcu_read_unlock();
758                 goto done;
759         }
760         rcu_read_unlock();
761
762         /*
763          * Continues from above, so we don't need an KERN_ level
764          */
765         printk(KERN_CONT " as a result of limit of %s\n", memcg_name);
766 done:
767
768         printk(KERN_INFO "memory: usage %llukB, limit %llukB, failcnt %llu\n",
769                 res_counter_read_u64(&memcg->res, RES_USAGE) >> 10,
770                 res_counter_read_u64(&memcg->res, RES_LIMIT) >> 10,
771                 res_counter_read_u64(&memcg->res, RES_FAILCNT));
772         printk(KERN_INFO "memory+swap: usage %llukB, limit %llukB, "
773                 "failcnt %llu\n",
774                 res_counter_read_u64(&memcg->memsw, RES_USAGE) >> 10,
775                 res_counter_read_u64(&memcg->memsw, RES_LIMIT) >> 10,
776                 res_counter_read_u64(&memcg->memsw, RES_FAILCNT));
777 }
778
779 /*
780  * This function returns the number of memcg under hierarchy tree. Returns
781  * 1(self count) if no children.
782  */
783 static int mem_cgroup_count_children(struct mem_cgroup *mem)
784 {
785         int num = 0;
786         mem_cgroup_walk_tree(mem, &num, mem_cgroup_count_children_cb);
787         return num;
788 }
789
790 /*
791  * Visit the first child (need not be the first child as per the ordering
792  * of the cgroup list, since we track last_scanned_child) of @mem and use
793  * that to reclaim free pages from.
794  */
795 static struct mem_cgroup *
796 mem_cgroup_select_victim(struct mem_cgroup *root_mem)
797 {
798         struct mem_cgroup *ret = NULL;
799         struct cgroup_subsys_state *css;
800         int nextid, found;
801
802         if (!root_mem->use_hierarchy) {
803                 css_get(&root_mem->css);
804                 ret = root_mem;
805         }
806
807         while (!ret) {
808                 rcu_read_lock();
809                 nextid = root_mem->last_scanned_child + 1;
810                 css = css_get_next(&mem_cgroup_subsys, nextid, &root_mem->css,
811                                    &found);
812                 if (css && css_tryget(css))
813                         ret = container_of(css, struct mem_cgroup, css);
814
815                 rcu_read_unlock();
816                 /* Updates scanning parameter */
817                 spin_lock(&root_mem->reclaim_param_lock);
818                 if (!css) {
819                         /* this means start scan from ID:1 */
820                         root_mem->last_scanned_child = 0;
821                 } else
822                         root_mem->last_scanned_child = found;
823                 spin_unlock(&root_mem->reclaim_param_lock);
824         }
825
826         return ret;
827 }
828
829 /*
830  * Scan the hierarchy if needed to reclaim memory. We remember the last child
831  * we reclaimed from, so that we don't end up penalizing one child extensively
832  * based on its position in the children list.
833  *
834  * root_mem is the original ancestor that we've been reclaim from.
835  *
836  * We give up and return to the caller when we visit root_mem twice.
837  * (other groups can be removed while we're walking....)
838  *
839  * If shrink==true, for avoiding to free too much, this returns immedieately.
840  */
841 static int mem_cgroup_hierarchical_reclaim(struct mem_cgroup *root_mem,
842                                    gfp_t gfp_mask, bool noswap, bool shrink)
843 {
844         struct mem_cgroup *victim;
845         int ret, total = 0;
846         int loop = 0;
847
848         while (loop < 2) {
849                 victim = mem_cgroup_select_victim(root_mem);
850                 if (victim == root_mem)
851                         loop++;
852                 if (!mem_cgroup_local_usage(&victim->stat)) {
853                         /* this cgroup's local usage == 0 */
854                         css_put(&victim->css);
855                         continue;
856                 }
857                 /* we use swappiness of local cgroup */
858                 ret = try_to_free_mem_cgroup_pages(victim, gfp_mask, noswap,
859                                                    get_swappiness(victim));
860                 css_put(&victim->css);
861                 /*
862                  * At shrinking usage, we can't check we should stop here or
863                  * reclaim more. It's depends on callers. last_scanned_child
864                  * will work enough for keeping fairness under tree.
865                  */
866                 if (shrink)
867                         return ret;
868                 total += ret;
869                 if (mem_cgroup_check_under_limit(root_mem))
870                         return 1 + total;
871         }
872         return total;
873 }
874
875 bool mem_cgroup_oom_called(struct task_struct *task)
876 {
877         bool ret = false;
878         struct mem_cgroup *mem;
879         struct mm_struct *mm;
880
881         rcu_read_lock();
882         mm = task->mm;
883         if (!mm)
884                 mm = &init_mm;
885         mem = mem_cgroup_from_task(rcu_dereference(mm->owner));
886         if (mem && time_before(jiffies, mem->last_oom_jiffies + HZ/10))
887                 ret = true;
888         rcu_read_unlock();
889         return ret;
890 }
891
892 static int record_last_oom_cb(struct mem_cgroup *mem, void *data)
893 {
894         mem->last_oom_jiffies = jiffies;
895         return 0;
896 }
897
898 static void record_last_oom(struct mem_cgroup *mem)
899 {
900         mem_cgroup_walk_tree(mem, NULL, record_last_oom_cb);
901 }
902
903
904 /*
905  * Unlike exported interface, "oom" parameter is added. if oom==true,
906  * oom-killer can be invoked.
907  */
908 static int __mem_cgroup_try_charge(struct mm_struct *mm,
909                         gfp_t gfp_mask, struct mem_cgroup **memcg,
910                         bool oom)
911 {
912         struct mem_cgroup *mem, *mem_over_limit;
913         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
914         struct res_counter *fail_res;
915
916         if (unlikely(test_thread_flag(TIF_MEMDIE))) {
917                 /* Don't account this! */
918                 *memcg = NULL;
919                 return 0;
920         }
921
922         /*
923          * We always charge the cgroup the mm_struct belongs to.
924          * The mm_struct's mem_cgroup changes on task migration if the
925          * thread group leader migrates. It's possible that mm is not
926          * set, if so charge the init_mm (happens for pagecache usage).
927          */
928         mem = *memcg;
929         if (likely(!mem)) {
930                 mem = try_get_mem_cgroup_from_mm(mm);
931                 *memcg = mem;
932         } else {
933                 css_get(&mem->css);
934         }
935         if (unlikely(!mem))
936                 return 0;
937
938         VM_BUG_ON(css_is_removed(&mem->css));
939
940         while (1) {
941                 int ret;
942                 bool noswap = false;
943
944                 ret = res_counter_charge(&mem->res, PAGE_SIZE, &fail_res);
945                 if (likely(!ret)) {
946                         if (!do_swap_account)
947                                 break;
948                         ret = res_counter_charge(&mem->memsw, PAGE_SIZE,
949                                                         &fail_res);
950                         if (likely(!ret))
951                                 break;
952                         /* mem+swap counter fails */
953                         res_counter_uncharge(&mem->res, PAGE_SIZE);
954                         noswap = true;
955                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
956                                                                         memsw);
957                 } else
958                         /* mem counter fails */
959                         mem_over_limit = mem_cgroup_from_res_counter(fail_res,
960                                                                         res);
961
962                 if (!(gfp_mask & __GFP_WAIT))
963                         goto nomem;
964
965                 ret = mem_cgroup_hierarchical_reclaim(mem_over_limit, gfp_mask,
966                                                         noswap, false);
967                 if (ret)
968                         continue;
969
970                 /*
971                  * try_to_free_mem_cgroup_pages() might not give us a full
972                  * picture of reclaim. Some pages are reclaimed and might be
973                  * moved to swap cache or just unmapped from the cgroup.
974                  * Check the limit again to see if the reclaim reduced the
975                  * current usage of the cgroup before giving up
976                  *
977                  */
978                 if (mem_cgroup_check_under_limit(mem_over_limit))
979                         continue;
980
981                 if (!nr_retries--) {
982                         if (oom) {
983                                 mutex_lock(&memcg_tasklist);
984                                 mem_cgroup_out_of_memory(mem_over_limit, gfp_mask);
985                                 mutex_unlock(&memcg_tasklist);
986                                 record_last_oom(mem_over_limit);
987                         }
988                         goto nomem;
989                 }
990         }
991         return 0;
992 nomem:
993         css_put(&mem->css);
994         return -ENOMEM;
995 }
996
997
998 /*
999  * A helper function to get mem_cgroup from ID. must be called under
1000  * rcu_read_lock(). The caller must check css_is_removed() or some if
1001  * it's concern. (dropping refcnt from swap can be called against removed
1002  * memcg.)
1003  */
1004 static struct mem_cgroup *mem_cgroup_lookup(unsigned short id)
1005 {
1006         struct cgroup_subsys_state *css;
1007
1008         /* ID 0 is unused ID */
1009         if (!id)
1010                 return NULL;
1011         css = css_lookup(&mem_cgroup_subsys, id);
1012         if (!css)
1013                 return NULL;
1014         return container_of(css, struct mem_cgroup, css);
1015 }
1016
1017 static struct mem_cgroup *try_get_mem_cgroup_from_swapcache(struct page *page)
1018 {
1019         struct mem_cgroup *mem;
1020         struct page_cgroup *pc;
1021         unsigned short id;
1022         swp_entry_t ent;
1023
1024         VM_BUG_ON(!PageLocked(page));
1025
1026         if (!PageSwapCache(page))
1027                 return NULL;
1028
1029         pc = lookup_page_cgroup(page);
1030         lock_page_cgroup(pc);
1031         if (PageCgroupUsed(pc)) {
1032                 mem = pc->mem_cgroup;
1033                 if (mem && !css_tryget(&mem->css))
1034                         mem = NULL;
1035         } else {
1036                 ent.val = page_private(page);
1037                 id = lookup_swap_cgroup(ent);
1038                 rcu_read_lock();
1039                 mem = mem_cgroup_lookup(id);
1040                 if (mem && !css_tryget(&mem->css))
1041                         mem = NULL;
1042                 rcu_read_unlock();
1043         }
1044         unlock_page_cgroup(pc);
1045         return mem;
1046 }
1047
1048 /*
1049  * commit a charge got by __mem_cgroup_try_charge() and makes page_cgroup to be
1050  * USED state. If already USED, uncharge and return.
1051  */
1052
1053 static void __mem_cgroup_commit_charge(struct mem_cgroup *mem,
1054                                      struct page_cgroup *pc,
1055                                      enum charge_type ctype)
1056 {
1057         /* try_charge() can return NULL to *memcg, taking care of it. */
1058         if (!mem)
1059                 return;
1060
1061         lock_page_cgroup(pc);
1062         if (unlikely(PageCgroupUsed(pc))) {
1063                 unlock_page_cgroup(pc);
1064                 res_counter_uncharge(&mem->res, PAGE_SIZE);
1065                 if (do_swap_account)
1066                         res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1067                 css_put(&mem->css);
1068                 return;
1069         }
1070         pc->mem_cgroup = mem;
1071         smp_wmb();
1072         pc->flags = pcg_default_flags[ctype];
1073
1074         mem_cgroup_charge_statistics(mem, pc, true);
1075
1076         unlock_page_cgroup(pc);
1077 }
1078
1079 /**
1080  * mem_cgroup_move_account - move account of the page
1081  * @pc: page_cgroup of the page.
1082  * @from: mem_cgroup which the page is moved from.
1083  * @to: mem_cgroup which the page is moved to. @from != @to.
1084  *
1085  * The caller must confirm following.
1086  * - page is not on LRU (isolate_page() is useful.)
1087  *
1088  * returns 0 at success,
1089  * returns -EBUSY when lock is busy or "pc" is unstable.
1090  *
1091  * This function does "uncharge" from old cgroup but doesn't do "charge" to
1092  * new cgroup. It should be done by a caller.
1093  */
1094
1095 static int mem_cgroup_move_account(struct page_cgroup *pc,
1096         struct mem_cgroup *from, struct mem_cgroup *to)
1097 {
1098         struct mem_cgroup_per_zone *from_mz, *to_mz;
1099         int nid, zid;
1100         int ret = -EBUSY;
1101
1102         VM_BUG_ON(from == to);
1103         VM_BUG_ON(PageLRU(pc->page));
1104
1105         nid = page_cgroup_nid(pc);
1106         zid = page_cgroup_zid(pc);
1107         from_mz =  mem_cgroup_zoneinfo(from, nid, zid);
1108         to_mz =  mem_cgroup_zoneinfo(to, nid, zid);
1109
1110         if (!trylock_page_cgroup(pc))
1111                 return ret;
1112
1113         if (!PageCgroupUsed(pc))
1114                 goto out;
1115
1116         if (pc->mem_cgroup != from)
1117                 goto out;
1118
1119         res_counter_uncharge(&from->res, PAGE_SIZE);
1120         mem_cgroup_charge_statistics(from, pc, false);
1121         if (do_swap_account)
1122                 res_counter_uncharge(&from->memsw, PAGE_SIZE);
1123         css_put(&from->css);
1124
1125         css_get(&to->css);
1126         pc->mem_cgroup = to;
1127         mem_cgroup_charge_statistics(to, pc, true);
1128         ret = 0;
1129 out:
1130         unlock_page_cgroup(pc);
1131         return ret;
1132 }
1133
1134 /*
1135  * move charges to its parent.
1136  */
1137
1138 static int mem_cgroup_move_parent(struct page_cgroup *pc,
1139                                   struct mem_cgroup *child,
1140                                   gfp_t gfp_mask)
1141 {
1142         struct page *page = pc->page;
1143         struct cgroup *cg = child->css.cgroup;
1144         struct cgroup *pcg = cg->parent;
1145         struct mem_cgroup *parent;
1146         int ret;
1147
1148         /* Is ROOT ? */
1149         if (!pcg)
1150                 return -EINVAL;
1151
1152
1153         parent = mem_cgroup_from_cont(pcg);
1154
1155
1156         ret = __mem_cgroup_try_charge(NULL, gfp_mask, &parent, false);
1157         if (ret || !parent)
1158                 return ret;
1159
1160         if (!get_page_unless_zero(page)) {
1161                 ret = -EBUSY;
1162                 goto uncharge;
1163         }
1164
1165         ret = isolate_lru_page(page);
1166
1167         if (ret)
1168                 goto cancel;
1169
1170         ret = mem_cgroup_move_account(pc, child, parent);
1171
1172         putback_lru_page(page);
1173         if (!ret) {
1174                 put_page(page);
1175                 /* drop extra refcnt by try_charge() */
1176                 css_put(&parent->css);
1177                 return 0;
1178         }
1179
1180 cancel:
1181         put_page(page);
1182 uncharge:
1183         /* drop extra refcnt by try_charge() */
1184         css_put(&parent->css);
1185         /* uncharge if move fails */
1186         res_counter_uncharge(&parent->res, PAGE_SIZE);
1187         if (do_swap_account)
1188                 res_counter_uncharge(&parent->memsw, PAGE_SIZE);
1189         return ret;
1190 }
1191
1192 /*
1193  * Charge the memory controller for page usage.
1194  * Return
1195  * 0 if the charge was successful
1196  * < 0 if the cgroup is over its limit
1197  */
1198 static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
1199                                 gfp_t gfp_mask, enum charge_type ctype,
1200                                 struct mem_cgroup *memcg)
1201 {
1202         struct mem_cgroup *mem;
1203         struct page_cgroup *pc;
1204         int ret;
1205
1206         pc = lookup_page_cgroup(page);
1207         /* can happen at boot */
1208         if (unlikely(!pc))
1209                 return 0;
1210         prefetchw(pc);
1211
1212         mem = memcg;
1213         ret = __mem_cgroup_try_charge(mm, gfp_mask, &mem, true);
1214         if (ret || !mem)
1215                 return ret;
1216
1217         __mem_cgroup_commit_charge(mem, pc, ctype);
1218         return 0;
1219 }
1220
1221 int mem_cgroup_newpage_charge(struct page *page,
1222                               struct mm_struct *mm, gfp_t gfp_mask)
1223 {
1224         if (mem_cgroup_disabled())
1225                 return 0;
1226         if (PageCompound(page))
1227                 return 0;
1228         /*
1229          * If already mapped, we don't have to account.
1230          * If page cache, page->mapping has address_space.
1231          * But page->mapping may have out-of-use anon_vma pointer,
1232          * detecit it by PageAnon() check. newly-mapped-anon's page->mapping
1233          * is NULL.
1234          */
1235         if (page_mapped(page) || (page->mapping && !PageAnon(page)))
1236                 return 0;
1237         if (unlikely(!mm))
1238                 mm = &init_mm;
1239         return mem_cgroup_charge_common(page, mm, gfp_mask,
1240                                 MEM_CGROUP_CHARGE_TYPE_MAPPED, NULL);
1241 }
1242
1243 static void
1244 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
1245                                         enum charge_type ctype);
1246
1247 int mem_cgroup_cache_charge(struct page *page, struct mm_struct *mm,
1248                                 gfp_t gfp_mask)
1249 {
1250         struct mem_cgroup *mem = NULL;
1251         int ret;
1252
1253         if (mem_cgroup_disabled())
1254                 return 0;
1255         if (PageCompound(page))
1256                 return 0;
1257         /*
1258          * Corner case handling. This is called from add_to_page_cache()
1259          * in usual. But some FS (shmem) precharges this page before calling it
1260          * and call add_to_page_cache() with GFP_NOWAIT.
1261          *
1262          * For GFP_NOWAIT case, the page may be pre-charged before calling
1263          * add_to_page_cache(). (See shmem.c) check it here and avoid to call
1264          * charge twice. (It works but has to pay a bit larger cost.)
1265          * And when the page is SwapCache, it should take swap information
1266          * into account. This is under lock_page() now.
1267          */
1268         if (!(gfp_mask & __GFP_WAIT)) {
1269                 struct page_cgroup *pc;
1270
1271
1272                 pc = lookup_page_cgroup(page);
1273                 if (!pc)
1274                         return 0;
1275                 lock_page_cgroup(pc);
1276                 if (PageCgroupUsed(pc)) {
1277                         unlock_page_cgroup(pc);
1278                         return 0;
1279                 }
1280                 unlock_page_cgroup(pc);
1281         }
1282
1283         if (unlikely(!mm && !mem))
1284                 mm = &init_mm;
1285
1286         if (page_is_file_cache(page))
1287                 return mem_cgroup_charge_common(page, mm, gfp_mask,
1288                                 MEM_CGROUP_CHARGE_TYPE_CACHE, NULL);
1289
1290         /* shmem */
1291         if (PageSwapCache(page)) {
1292                 ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
1293                 if (!ret)
1294                         __mem_cgroup_commit_charge_swapin(page, mem,
1295                                         MEM_CGROUP_CHARGE_TYPE_SHMEM);
1296         } else
1297                 ret = mem_cgroup_charge_common(page, mm, gfp_mask,
1298                                         MEM_CGROUP_CHARGE_TYPE_SHMEM, mem);
1299
1300         return ret;
1301 }
1302
1303 /*
1304  * While swap-in, try_charge -> commit or cancel, the page is locked.
1305  * And when try_charge() successfully returns, one refcnt to memcg without
1306  * struct page_cgroup is aquired. This refcnt will be cumsumed by
1307  * "commit()" or removed by "cancel()"
1308  */
1309 int mem_cgroup_try_charge_swapin(struct mm_struct *mm,
1310                                  struct page *page,
1311                                  gfp_t mask, struct mem_cgroup **ptr)
1312 {
1313         struct mem_cgroup *mem;
1314         int ret;
1315
1316         if (mem_cgroup_disabled())
1317                 return 0;
1318
1319         if (!do_swap_account)
1320                 goto charge_cur_mm;
1321         /*
1322          * A racing thread's fault, or swapoff, may have already updated
1323          * the pte, and even removed page from swap cache: return success
1324          * to go on to do_swap_page()'s pte_same() test, which should fail.
1325          */
1326         if (!PageSwapCache(page))
1327                 return 0;
1328         mem = try_get_mem_cgroup_from_swapcache(page);
1329         if (!mem)
1330                 goto charge_cur_mm;
1331         *ptr = mem;
1332         ret = __mem_cgroup_try_charge(NULL, mask, ptr, true);
1333         /* drop extra refcnt from tryget */
1334         css_put(&mem->css);
1335         return ret;
1336 charge_cur_mm:
1337         if (unlikely(!mm))
1338                 mm = &init_mm;
1339         return __mem_cgroup_try_charge(mm, mask, ptr, true);
1340 }
1341
1342 static void
1343 __mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr,
1344                                         enum charge_type ctype)
1345 {
1346         struct page_cgroup *pc;
1347
1348         if (mem_cgroup_disabled())
1349                 return;
1350         if (!ptr)
1351                 return;
1352         pc = lookup_page_cgroup(page);
1353         mem_cgroup_lru_del_before_commit_swapcache(page);
1354         __mem_cgroup_commit_charge(ptr, pc, ctype);
1355         mem_cgroup_lru_add_after_commit_swapcache(page);
1356         /*
1357          * Now swap is on-memory. This means this page may be
1358          * counted both as mem and swap....double count.
1359          * Fix it by uncharging from memsw. Basically, this SwapCache is stable
1360          * under lock_page(). But in do_swap_page()::memory.c, reuse_swap_page()
1361          * may call delete_from_swap_cache() before reach here.
1362          */
1363         if (do_swap_account && PageSwapCache(page)) {
1364                 swp_entry_t ent = {.val = page_private(page)};
1365                 unsigned short id;
1366                 struct mem_cgroup *memcg;
1367
1368                 id = swap_cgroup_record(ent, 0);
1369                 rcu_read_lock();
1370                 memcg = mem_cgroup_lookup(id);
1371                 if (memcg) {
1372                         /*
1373                          * This recorded memcg can be obsolete one. So, avoid
1374                          * calling css_tryget
1375                          */
1376                         res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1377                         mem_cgroup_put(memcg);
1378                 }
1379                 rcu_read_unlock();
1380         }
1381         /* add this page(page_cgroup) to the LRU we want. */
1382
1383 }
1384
1385 void mem_cgroup_commit_charge_swapin(struct page *page, struct mem_cgroup *ptr)
1386 {
1387         __mem_cgroup_commit_charge_swapin(page, ptr,
1388                                         MEM_CGROUP_CHARGE_TYPE_MAPPED);
1389 }
1390
1391 void mem_cgroup_cancel_charge_swapin(struct mem_cgroup *mem)
1392 {
1393         if (mem_cgroup_disabled())
1394                 return;
1395         if (!mem)
1396                 return;
1397         res_counter_uncharge(&mem->res, PAGE_SIZE);
1398         if (do_swap_account)
1399                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1400         css_put(&mem->css);
1401 }
1402
1403
1404 /*
1405  * uncharge if !page_mapped(page)
1406  */
1407 static struct mem_cgroup *
1408 __mem_cgroup_uncharge_common(struct page *page, enum charge_type ctype)
1409 {
1410         struct page_cgroup *pc;
1411         struct mem_cgroup *mem = NULL;
1412         struct mem_cgroup_per_zone *mz;
1413
1414         if (mem_cgroup_disabled())
1415                 return NULL;
1416
1417         if (PageSwapCache(page))
1418                 return NULL;
1419
1420         /*
1421          * Check if our page_cgroup is valid
1422          */
1423         pc = lookup_page_cgroup(page);
1424         if (unlikely(!pc || !PageCgroupUsed(pc)))
1425                 return NULL;
1426
1427         lock_page_cgroup(pc);
1428
1429         mem = pc->mem_cgroup;
1430
1431         if (!PageCgroupUsed(pc))
1432                 goto unlock_out;
1433
1434         switch (ctype) {
1435         case MEM_CGROUP_CHARGE_TYPE_MAPPED:
1436                 if (page_mapped(page))
1437                         goto unlock_out;
1438                 break;
1439         case MEM_CGROUP_CHARGE_TYPE_SWAPOUT:
1440                 if (!PageAnon(page)) {  /* Shared memory */
1441                         if (page->mapping && !page_is_file_cache(page))
1442                                 goto unlock_out;
1443                 } else if (page_mapped(page)) /* Anon */
1444                                 goto unlock_out;
1445                 break;
1446         default:
1447                 break;
1448         }
1449
1450         res_counter_uncharge(&mem->res, PAGE_SIZE);
1451         if (do_swap_account && (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT))
1452                 res_counter_uncharge(&mem->memsw, PAGE_SIZE);
1453         mem_cgroup_charge_statistics(mem, pc, false);
1454
1455         ClearPageCgroupUsed(pc);
1456         /*
1457          * pc->mem_cgroup is not cleared here. It will be accessed when it's
1458          * freed from LRU. This is safe because uncharged page is expected not
1459          * to be reused (freed soon). Exception is SwapCache, it's handled by
1460          * special functions.
1461          */
1462
1463         mz = page_cgroup_zoneinfo(pc);
1464         unlock_page_cgroup(pc);
1465
1466         /* at swapout, this memcg will be accessed to record to swap */
1467         if (ctype != MEM_CGROUP_CHARGE_TYPE_SWAPOUT)
1468                 css_put(&mem->css);
1469
1470         return mem;
1471
1472 unlock_out:
1473         unlock_page_cgroup(pc);
1474         return NULL;
1475 }
1476
1477 void mem_cgroup_uncharge_page(struct page *page)
1478 {
1479         /* early check. */
1480         if (page_mapped(page))
1481                 return;
1482         if (page->mapping && !PageAnon(page))
1483                 return;
1484         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_MAPPED);
1485 }
1486
1487 void mem_cgroup_uncharge_cache_page(struct page *page)
1488 {
1489         VM_BUG_ON(page_mapped(page));
1490         VM_BUG_ON(page->mapping);
1491         __mem_cgroup_uncharge_common(page, MEM_CGROUP_CHARGE_TYPE_CACHE);
1492 }
1493
1494 #ifdef CONFIG_SWAP
1495 /*
1496  * called after __delete_from_swap_cache() and drop "page" account.
1497  * memcg information is recorded to swap_cgroup of "ent"
1498  */
1499 void mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent)
1500 {
1501         struct mem_cgroup *memcg;
1502
1503         memcg = __mem_cgroup_uncharge_common(page,
1504                                         MEM_CGROUP_CHARGE_TYPE_SWAPOUT);
1505         /* record memcg information */
1506         if (do_swap_account && memcg) {
1507                 swap_cgroup_record(ent, css_id(&memcg->css));
1508                 mem_cgroup_get(memcg);
1509         }
1510         if (memcg)
1511                 css_put(&memcg->css);
1512 }
1513 #endif
1514
1515 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
1516 /*
1517  * called from swap_entry_free(). remove record in swap_cgroup and
1518  * uncharge "memsw" account.
1519  */
1520 void mem_cgroup_uncharge_swap(swp_entry_t ent)
1521 {
1522         struct mem_cgroup *memcg;
1523         unsigned short id;
1524
1525         if (!do_swap_account)
1526                 return;
1527
1528         id = swap_cgroup_record(ent, 0);
1529         rcu_read_lock();
1530         memcg = mem_cgroup_lookup(id);
1531         if (memcg) {
1532                 /*
1533                  * We uncharge this because swap is freed.
1534                  * This memcg can be obsolete one. We avoid calling css_tryget
1535                  */
1536                 res_counter_uncharge(&memcg->memsw, PAGE_SIZE);
1537                 mem_cgroup_put(memcg);
1538         }
1539         rcu_read_unlock();
1540 }
1541 #endif
1542
1543 /*
1544  * Before starting migration, account PAGE_SIZE to mem_cgroup that the old
1545  * page belongs to.
1546  */
1547 int mem_cgroup_prepare_migration(struct page *page, struct mem_cgroup **ptr)
1548 {
1549         struct page_cgroup *pc;
1550         struct mem_cgroup *mem = NULL;
1551         int ret = 0;
1552
1553         if (mem_cgroup_disabled())
1554                 return 0;
1555
1556         pc = lookup_page_cgroup(page);
1557         lock_page_cgroup(pc);
1558         if (PageCgroupUsed(pc)) {
1559                 mem = pc->mem_cgroup;
1560                 css_get(&mem->css);
1561         }
1562         unlock_page_cgroup(pc);
1563
1564         if (mem) {
1565                 ret = __mem_cgroup_try_charge(NULL, GFP_KERNEL, &mem, false);
1566                 css_put(&mem->css);
1567         }
1568         *ptr = mem;
1569         return ret;
1570 }
1571
1572 /* remove redundant charge if migration failed*/
1573 void mem_cgroup_end_migration(struct mem_cgroup *mem,
1574                 struct page *oldpage, struct page *newpage)
1575 {
1576         struct page *target, *unused;
1577         struct page_cgroup *pc;
1578         enum charge_type ctype;
1579
1580         if (!mem)
1581                 return;
1582
1583         /* at migration success, oldpage->mapping is NULL. */
1584         if (oldpage->mapping) {
1585                 target = oldpage;
1586                 unused = NULL;
1587         } else {
1588                 target = newpage;
1589                 unused = oldpage;
1590         }
1591
1592         if (PageAnon(target))
1593                 ctype = MEM_CGROUP_CHARGE_TYPE_MAPPED;
1594         else if (page_is_file_cache(target))
1595                 ctype = MEM_CGROUP_CHARGE_TYPE_CACHE;
1596         else
1597                 ctype = MEM_CGROUP_CHARGE_TYPE_SHMEM;
1598
1599         /* unused page is not on radix-tree now. */
1600         if (unused)
1601                 __mem_cgroup_uncharge_common(unused, ctype);
1602
1603         pc = lookup_page_cgroup(target);
1604         /*
1605          * __mem_cgroup_commit_charge() check PCG_USED bit of page_cgroup.
1606          * So, double-counting is effectively avoided.
1607          */
1608         __mem_cgroup_commit_charge(mem, pc, ctype);
1609
1610         /*
1611          * Both of oldpage and newpage are still under lock_page().
1612          * Then, we don't have to care about race in radix-tree.
1613          * But we have to be careful that this page is unmapped or not.
1614          *
1615          * There is a case for !page_mapped(). At the start of
1616          * migration, oldpage was mapped. But now, it's zapped.
1617          * But we know *target* page is not freed/reused under us.
1618          * mem_cgroup_uncharge_page() does all necessary checks.
1619          */
1620         if (ctype == MEM_CGROUP_CHARGE_TYPE_MAPPED)
1621                 mem_cgroup_uncharge_page(target);
1622 }
1623
1624 /*
1625  * A call to try to shrink memory usage on charge failure at shmem's swapin.
1626  * Calling hierarchical_reclaim is not enough because we should update
1627  * last_oom_jiffies to prevent pagefault_out_of_memory from invoking global OOM.
1628  * Moreover considering hierarchy, we should reclaim from the mem_over_limit,
1629  * not from the memcg which this page would be charged to.
1630  * try_charge_swapin does all of these works properly.
1631  */
1632 int mem_cgroup_shmem_charge_fallback(struct page *page,
1633                             struct mm_struct *mm,
1634                             gfp_t gfp_mask)
1635 {
1636         struct mem_cgroup *mem = NULL;
1637         int ret;
1638
1639         if (mem_cgroup_disabled())
1640                 return 0;
1641
1642         ret = mem_cgroup_try_charge_swapin(mm, page, gfp_mask, &mem);
1643         if (!ret)
1644                 mem_cgroup_cancel_charge_swapin(mem); /* it does !mem check */
1645
1646         return ret;
1647 }
1648
1649 static DEFINE_MUTEX(set_limit_mutex);
1650
1651 static int mem_cgroup_resize_limit(struct mem_cgroup *memcg,
1652                                 unsigned long long val)
1653 {
1654         int retry_count;
1655         int progress;
1656         u64 memswlimit;
1657         int ret = 0;
1658         int children = mem_cgroup_count_children(memcg);
1659         u64 curusage, oldusage;
1660
1661         /*
1662          * For keeping hierarchical_reclaim simple, how long we should retry
1663          * is depends on callers. We set our retry-count to be function
1664          * of # of children which we should visit in this loop.
1665          */
1666         retry_count = MEM_CGROUP_RECLAIM_RETRIES * children;
1667
1668         oldusage = res_counter_read_u64(&memcg->res, RES_USAGE);
1669
1670         while (retry_count) {
1671                 if (signal_pending(current)) {
1672                         ret = -EINTR;
1673                         break;
1674                 }
1675                 /*
1676                  * Rather than hide all in some function, I do this in
1677                  * open coded manner. You see what this really does.
1678                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1679                  */
1680                 mutex_lock(&set_limit_mutex);
1681                 memswlimit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1682                 if (memswlimit < val) {
1683                         ret = -EINVAL;
1684                         mutex_unlock(&set_limit_mutex);
1685                         break;
1686                 }
1687                 ret = res_counter_set_limit(&memcg->res, val);
1688                 mutex_unlock(&set_limit_mutex);
1689
1690                 if (!ret)
1691                         break;
1692
1693                 progress = mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL,
1694                                                    false, true);
1695                 curusage = res_counter_read_u64(&memcg->res, RES_USAGE);
1696                 /* Usage is reduced ? */
1697                 if (curusage >= oldusage)
1698                         retry_count--;
1699                 else
1700                         oldusage = curusage;
1701         }
1702
1703         return ret;
1704 }
1705
1706 int mem_cgroup_resize_memsw_limit(struct mem_cgroup *memcg,
1707                                 unsigned long long val)
1708 {
1709         int retry_count;
1710         u64 memlimit, oldusage, curusage;
1711         int children = mem_cgroup_count_children(memcg);
1712         int ret = -EBUSY;
1713
1714         if (!do_swap_account)
1715                 return -EINVAL;
1716         /* see mem_cgroup_resize_res_limit */
1717         retry_count = children * MEM_CGROUP_RECLAIM_RETRIES;
1718         oldusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1719         while (retry_count) {
1720                 if (signal_pending(current)) {
1721                         ret = -EINTR;
1722                         break;
1723                 }
1724                 /*
1725                  * Rather than hide all in some function, I do this in
1726                  * open coded manner. You see what this really does.
1727                  * We have to guarantee mem->res.limit < mem->memsw.limit.
1728                  */
1729                 mutex_lock(&set_limit_mutex);
1730                 memlimit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1731                 if (memlimit > val) {
1732                         ret = -EINVAL;
1733                         mutex_unlock(&set_limit_mutex);
1734                         break;
1735                 }
1736                 ret = res_counter_set_limit(&memcg->memsw, val);
1737                 mutex_unlock(&set_limit_mutex);
1738
1739                 if (!ret)
1740                         break;
1741
1742                 mem_cgroup_hierarchical_reclaim(memcg, GFP_KERNEL, true, true);
1743                 curusage = res_counter_read_u64(&memcg->memsw, RES_USAGE);
1744                 /* Usage is reduced ? */
1745                 if (curusage >= oldusage)
1746                         retry_count--;
1747                 else
1748                         oldusage = curusage;
1749         }
1750         return ret;
1751 }
1752
1753 /*
1754  * This routine traverse page_cgroup in given list and drop them all.
1755  * *And* this routine doesn't reclaim page itself, just removes page_cgroup.
1756  */
1757 static int mem_cgroup_force_empty_list(struct mem_cgroup *mem,
1758                                 int node, int zid, enum lru_list lru)
1759 {
1760         struct zone *zone;
1761         struct mem_cgroup_per_zone *mz;
1762         struct page_cgroup *pc, *busy;
1763         unsigned long flags, loop;
1764         struct list_head *list;
1765         int ret = 0;
1766
1767         zone = &NODE_DATA(node)->node_zones[zid];
1768         mz = mem_cgroup_zoneinfo(mem, node, zid);
1769         list = &mz->lists[lru];
1770
1771         loop = MEM_CGROUP_ZSTAT(mz, lru);
1772         /* give some margin against EBUSY etc...*/
1773         loop += 256;
1774         busy = NULL;
1775         while (loop--) {
1776                 ret = 0;
1777                 spin_lock_irqsave(&zone->lru_lock, flags);
1778                 if (list_empty(list)) {
1779                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1780                         break;
1781                 }
1782                 pc = list_entry(list->prev, struct page_cgroup, lru);
1783                 if (busy == pc) {
1784                         list_move(&pc->lru, list);
1785                         busy = 0;
1786                         spin_unlock_irqrestore(&zone->lru_lock, flags);
1787                         continue;
1788                 }
1789                 spin_unlock_irqrestore(&zone->lru_lock, flags);
1790
1791                 ret = mem_cgroup_move_parent(pc, mem, GFP_KERNEL);
1792                 if (ret == -ENOMEM)
1793                         break;
1794
1795                 if (ret == -EBUSY || ret == -EINVAL) {
1796                         /* found lock contention or "pc" is obsolete. */
1797                         busy = pc;
1798                         cond_resched();
1799                 } else
1800                         busy = NULL;
1801         }
1802
1803         if (!ret && !list_empty(list))
1804                 return -EBUSY;
1805         return ret;
1806 }
1807
1808 /*
1809  * make mem_cgroup's charge to be 0 if there is no task.
1810  * This enables deleting this mem_cgroup.
1811  */
1812 static int mem_cgroup_force_empty(struct mem_cgroup *mem, bool free_all)
1813 {
1814         int ret;
1815         int node, zid, shrink;
1816         int nr_retries = MEM_CGROUP_RECLAIM_RETRIES;
1817         struct cgroup *cgrp = mem->css.cgroup;
1818
1819         css_get(&mem->css);
1820
1821         shrink = 0;
1822         /* should free all ? */
1823         if (free_all)
1824                 goto try_to_free;
1825 move_account:
1826         while (mem->res.usage > 0) {
1827                 ret = -EBUSY;
1828                 if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children))
1829                         goto out;
1830                 ret = -EINTR;
1831                 if (signal_pending(current))
1832                         goto out;
1833                 /* This is for making all *used* pages to be on LRU. */
1834                 lru_add_drain_all();
1835                 ret = 0;
1836                 for_each_node_state(node, N_HIGH_MEMORY) {
1837                         for (zid = 0; !ret && zid < MAX_NR_ZONES; zid++) {
1838                                 enum lru_list l;
1839                                 for_each_lru(l) {
1840                                         ret = mem_cgroup_force_empty_list(mem,
1841                                                         node, zid, l);
1842                                         if (ret)
1843                                                 break;
1844                                 }
1845                         }
1846                         if (ret)
1847                                 break;
1848                 }
1849                 /* it seems parent cgroup doesn't have enough mem */
1850                 if (ret == -ENOMEM)
1851                         goto try_to_free;
1852                 cond_resched();
1853         }
1854         ret = 0;
1855 out:
1856         css_put(&mem->css);
1857         return ret;
1858
1859 try_to_free:
1860         /* returns EBUSY if there is a task or if we come here twice. */
1861         if (cgroup_task_count(cgrp) || !list_empty(&cgrp->children) || shrink) {
1862                 ret = -EBUSY;
1863                 goto out;
1864         }
1865         /* we call try-to-free pages for make this cgroup empty */
1866         lru_add_drain_all();
1867         /* try to free all pages in this cgroup */
1868         shrink = 1;
1869         while (nr_retries && mem->res.usage > 0) {
1870                 int progress;
1871
1872                 if (signal_pending(current)) {
1873                         ret = -EINTR;
1874                         goto out;
1875                 }
1876                 progress = try_to_free_mem_cgroup_pages(mem, GFP_KERNEL,
1877                                                 false, get_swappiness(mem));
1878                 if (!progress) {
1879                         nr_retries--;
1880                         /* maybe some writeback is necessary */
1881                         congestion_wait(WRITE, HZ/10);
1882                 }
1883
1884         }
1885         lru_add_drain();
1886         /* try move_account...there may be some *locked* pages. */
1887         if (mem->res.usage)
1888                 goto move_account;
1889         ret = 0;
1890         goto out;
1891 }
1892
1893 int mem_cgroup_force_empty_write(struct cgroup *cont, unsigned int event)
1894 {
1895         return mem_cgroup_force_empty(mem_cgroup_from_cont(cont), true);
1896 }
1897
1898
1899 static u64 mem_cgroup_hierarchy_read(struct cgroup *cont, struct cftype *cft)
1900 {
1901         return mem_cgroup_from_cont(cont)->use_hierarchy;
1902 }
1903
1904 static int mem_cgroup_hierarchy_write(struct cgroup *cont, struct cftype *cft,
1905                                         u64 val)
1906 {
1907         int retval = 0;
1908         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1909         struct cgroup *parent = cont->parent;
1910         struct mem_cgroup *parent_mem = NULL;
1911
1912         if (parent)
1913                 parent_mem = mem_cgroup_from_cont(parent);
1914
1915         cgroup_lock();
1916         /*
1917          * If parent's use_hiearchy is set, we can't make any modifications
1918          * in the child subtrees. If it is unset, then the change can
1919          * occur, provided the current cgroup has no children.
1920          *
1921          * For the root cgroup, parent_mem is NULL, we allow value to be
1922          * set if there are no children.
1923          */
1924         if ((!parent_mem || !parent_mem->use_hierarchy) &&
1925                                 (val == 1 || val == 0)) {
1926                 if (list_empty(&cont->children))
1927                         mem->use_hierarchy = val;
1928                 else
1929                         retval = -EBUSY;
1930         } else
1931                 retval = -EINVAL;
1932         cgroup_unlock();
1933
1934         return retval;
1935 }
1936
1937 static u64 mem_cgroup_read(struct cgroup *cont, struct cftype *cft)
1938 {
1939         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
1940         u64 val = 0;
1941         int type, name;
1942
1943         type = MEMFILE_TYPE(cft->private);
1944         name = MEMFILE_ATTR(cft->private);
1945         switch (type) {
1946         case _MEM:
1947                 val = res_counter_read_u64(&mem->res, name);
1948                 break;
1949         case _MEMSWAP:
1950                 if (do_swap_account)
1951                         val = res_counter_read_u64(&mem->memsw, name);
1952                 break;
1953         default:
1954                 BUG();
1955                 break;
1956         }
1957         return val;
1958 }
1959 /*
1960  * The user of this function is...
1961  * RES_LIMIT.
1962  */
1963 static int mem_cgroup_write(struct cgroup *cont, struct cftype *cft,
1964                             const char *buffer)
1965 {
1966         struct mem_cgroup *memcg = mem_cgroup_from_cont(cont);
1967         int type, name;
1968         unsigned long long val;
1969         int ret;
1970
1971         type = MEMFILE_TYPE(cft->private);
1972         name = MEMFILE_ATTR(cft->private);
1973         switch (name) {
1974         case RES_LIMIT:
1975                 /* This function does all necessary parse...reuse it */
1976                 ret = res_counter_memparse_write_strategy(buffer, &val);
1977                 if (ret)
1978                         break;
1979                 if (type == _MEM)
1980                         ret = mem_cgroup_resize_limit(memcg, val);
1981                 else
1982                         ret = mem_cgroup_resize_memsw_limit(memcg, val);
1983                 break;
1984         default:
1985                 ret = -EINVAL; /* should be BUG() ? */
1986                 break;
1987         }
1988         return ret;
1989 }
1990
1991 static void memcg_get_hierarchical_limit(struct mem_cgroup *memcg,
1992                 unsigned long long *mem_limit, unsigned long long *memsw_limit)
1993 {
1994         struct cgroup *cgroup;
1995         unsigned long long min_limit, min_memsw_limit, tmp;
1996
1997         min_limit = res_counter_read_u64(&memcg->res, RES_LIMIT);
1998         min_memsw_limit = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
1999         cgroup = memcg->css.cgroup;
2000         if (!memcg->use_hierarchy)
2001                 goto out;
2002
2003         while (cgroup->parent) {
2004                 cgroup = cgroup->parent;
2005                 memcg = mem_cgroup_from_cont(cgroup);
2006                 if (!memcg->use_hierarchy)
2007                         break;
2008                 tmp = res_counter_read_u64(&memcg->res, RES_LIMIT);
2009                 min_limit = min(min_limit, tmp);
2010                 tmp = res_counter_read_u64(&memcg->memsw, RES_LIMIT);
2011                 min_memsw_limit = min(min_memsw_limit, tmp);
2012         }
2013 out:
2014         *mem_limit = min_limit;
2015         *memsw_limit = min_memsw_limit;
2016         return;
2017 }
2018
2019 static int mem_cgroup_reset(struct cgroup *cont, unsigned int event)
2020 {
2021         struct mem_cgroup *mem;
2022         int type, name;
2023
2024         mem = mem_cgroup_from_cont(cont);
2025         type = MEMFILE_TYPE(event);
2026         name = MEMFILE_ATTR(event);
2027         switch (name) {
2028         case RES_MAX_USAGE:
2029                 if (type == _MEM)
2030                         res_counter_reset_max(&mem->res);
2031                 else
2032                         res_counter_reset_max(&mem->memsw);
2033                 break;
2034         case RES_FAILCNT:
2035                 if (type == _MEM)
2036                         res_counter_reset_failcnt(&mem->res);
2037                 else
2038                         res_counter_reset_failcnt(&mem->memsw);
2039                 break;
2040         }
2041         return 0;
2042 }
2043
2044
2045 /* For read statistics */
2046 enum {
2047         MCS_CACHE,
2048         MCS_RSS,
2049         MCS_PGPGIN,
2050         MCS_PGPGOUT,
2051         MCS_INACTIVE_ANON,
2052         MCS_ACTIVE_ANON,
2053         MCS_INACTIVE_FILE,
2054         MCS_ACTIVE_FILE,
2055         MCS_UNEVICTABLE,
2056         NR_MCS_STAT,
2057 };
2058
2059 struct mcs_total_stat {
2060         s64 stat[NR_MCS_STAT];
2061 };
2062
2063 struct {
2064         char *local_name;
2065         char *total_name;
2066 } memcg_stat_strings[NR_MCS_STAT] = {
2067         {"cache", "total_cache"},
2068         {"rss", "total_rss"},
2069         {"pgpgin", "total_pgpgin"},
2070         {"pgpgout", "total_pgpgout"},
2071         {"inactive_anon", "total_inactive_anon"},
2072         {"active_anon", "total_active_anon"},
2073         {"inactive_file", "total_inactive_file"},
2074         {"active_file", "total_active_file"},
2075         {"unevictable", "total_unevictable"}
2076 };
2077
2078
2079 static int mem_cgroup_get_local_stat(struct mem_cgroup *mem, void *data)
2080 {
2081         struct mcs_total_stat *s = data;
2082         s64 val;
2083
2084         /* per cpu stat */
2085         val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_CACHE);
2086         s->stat[MCS_CACHE] += val * PAGE_SIZE;
2087         val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_RSS);
2088         s->stat[MCS_RSS] += val * PAGE_SIZE;
2089         val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGIN_COUNT);
2090         s->stat[MCS_PGPGIN] += val;
2091         val = mem_cgroup_read_stat(&mem->stat, MEM_CGROUP_STAT_PGPGOUT_COUNT);
2092         s->stat[MCS_PGPGOUT] += val;
2093
2094         /* per zone stat */
2095         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_ANON);
2096         s->stat[MCS_INACTIVE_ANON] += val * PAGE_SIZE;
2097         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_ANON);
2098         s->stat[MCS_ACTIVE_ANON] += val * PAGE_SIZE;
2099         val = mem_cgroup_get_local_zonestat(mem, LRU_INACTIVE_FILE);
2100         s->stat[MCS_INACTIVE_FILE] += val * PAGE_SIZE;
2101         val = mem_cgroup_get_local_zonestat(mem, LRU_ACTIVE_FILE);
2102         s->stat[MCS_ACTIVE_FILE] += val * PAGE_SIZE;
2103         val = mem_cgroup_get_local_zonestat(mem, LRU_UNEVICTABLE);
2104         s->stat[MCS_UNEVICTABLE] += val * PAGE_SIZE;
2105         return 0;
2106 }
2107
2108 static void
2109 mem_cgroup_get_total_stat(struct mem_cgroup *mem, struct mcs_total_stat *s)
2110 {
2111         mem_cgroup_walk_tree(mem, s, mem_cgroup_get_local_stat);
2112 }
2113
2114 static int mem_control_stat_show(struct cgroup *cont, struct cftype *cft,
2115                                  struct cgroup_map_cb *cb)
2116 {
2117         struct mem_cgroup *mem_cont = mem_cgroup_from_cont(cont);
2118         struct mcs_total_stat mystat;
2119         int i;
2120
2121         memset(&mystat, 0, sizeof(mystat));
2122         mem_cgroup_get_local_stat(mem_cont, &mystat);
2123
2124         for (i = 0; i < NR_MCS_STAT; i++)
2125                 cb->fill(cb, memcg_stat_strings[i].local_name, mystat.stat[i]);
2126
2127         /* Hierarchical information */
2128         {
2129                 unsigned long long limit, memsw_limit;
2130                 memcg_get_hierarchical_limit(mem_cont, &limit, &memsw_limit);
2131                 cb->fill(cb, "hierarchical_memory_limit", limit);
2132                 if (do_swap_account)
2133                         cb->fill(cb, "hierarchical_memsw_limit", memsw_limit);
2134         }
2135
2136         memset(&mystat, 0, sizeof(mystat));
2137         mem_cgroup_get_total_stat(mem_cont, &mystat);
2138         for (i = 0; i < NR_MCS_STAT; i++)
2139                 cb->fill(cb, memcg_stat_strings[i].total_name, mystat.stat[i]);
2140
2141
2142 #ifdef CONFIG_DEBUG_VM
2143         cb->fill(cb, "inactive_ratio", calc_inactive_ratio(mem_cont, NULL));
2144
2145         {
2146                 int nid, zid;
2147                 struct mem_cgroup_per_zone *mz;
2148                 unsigned long recent_rotated[2] = {0, 0};
2149                 unsigned long recent_scanned[2] = {0, 0};
2150
2151                 for_each_online_node(nid)
2152                         for (zid = 0; zid < MAX_NR_ZONES; zid++) {
2153                                 mz = mem_cgroup_zoneinfo(mem_cont, nid, zid);
2154
2155                                 recent_rotated[0] +=
2156                                         mz->reclaim_stat.recent_rotated[0];
2157                                 recent_rotated[1] +=
2158                                         mz->reclaim_stat.recent_rotated[1];
2159                                 recent_scanned[0] +=
2160                                         mz->reclaim_stat.recent_scanned[0];
2161                                 recent_scanned[1] +=
2162                                         mz->reclaim_stat.recent_scanned[1];
2163                         }
2164                 cb->fill(cb, "recent_rotated_anon", recent_rotated[0]);
2165                 cb->fill(cb, "recent_rotated_file", recent_rotated[1]);
2166                 cb->fill(cb, "recent_scanned_anon", recent_scanned[0]);
2167                 cb->fill(cb, "recent_scanned_file", recent_scanned[1]);
2168         }
2169 #endif
2170
2171         return 0;
2172 }
2173
2174 static u64 mem_cgroup_swappiness_read(struct cgroup *cgrp, struct cftype *cft)
2175 {
2176         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
2177
2178         return get_swappiness(memcg);
2179 }
2180
2181 static int mem_cgroup_swappiness_write(struct cgroup *cgrp, struct cftype *cft,
2182                                        u64 val)
2183 {
2184         struct mem_cgroup *memcg = mem_cgroup_from_cont(cgrp);
2185         struct mem_cgroup *parent;
2186
2187         if (val > 100)
2188                 return -EINVAL;
2189
2190         if (cgrp->parent == NULL)
2191                 return -EINVAL;
2192
2193         parent = mem_cgroup_from_cont(cgrp->parent);
2194
2195         cgroup_lock();
2196
2197         /* If under hierarchy, only empty-root can set this value */
2198         if ((parent->use_hierarchy) ||
2199             (memcg->use_hierarchy && !list_empty(&cgrp->children))) {
2200                 cgroup_unlock();
2201                 return -EINVAL;
2202         }
2203
2204         spin_lock(&memcg->reclaim_param_lock);
2205         memcg->swappiness = val;
2206         spin_unlock(&memcg->reclaim_param_lock);
2207
2208         cgroup_unlock();
2209
2210         return 0;
2211 }
2212
2213
2214 static struct cftype mem_cgroup_files[] = {
2215         {
2216                 .name = "usage_in_bytes",
2217                 .private = MEMFILE_PRIVATE(_MEM, RES_USAGE),
2218                 .read_u64 = mem_cgroup_read,
2219         },
2220         {
2221                 .name = "max_usage_in_bytes",
2222                 .private = MEMFILE_PRIVATE(_MEM, RES_MAX_USAGE),
2223                 .trigger = mem_cgroup_reset,
2224                 .read_u64 = mem_cgroup_read,
2225         },
2226         {
2227                 .name = "limit_in_bytes",
2228                 .private = MEMFILE_PRIVATE(_MEM, RES_LIMIT),
2229                 .write_string = mem_cgroup_write,
2230                 .read_u64 = mem_cgroup_read,
2231         },
2232         {
2233                 .name = "failcnt",
2234                 .private = MEMFILE_PRIVATE(_MEM, RES_FAILCNT),
2235                 .trigger = mem_cgroup_reset,
2236                 .read_u64 = mem_cgroup_read,
2237         },
2238         {
2239                 .name = "stat",
2240                 .read_map = mem_control_stat_show,
2241         },
2242         {
2243                 .name = "force_empty",
2244                 .trigger = mem_cgroup_force_empty_write,
2245         },
2246         {
2247                 .name = "use_hierarchy",
2248                 .write_u64 = mem_cgroup_hierarchy_write,
2249                 .read_u64 = mem_cgroup_hierarchy_read,
2250         },
2251         {
2252                 .name = "swappiness",
2253                 .read_u64 = mem_cgroup_swappiness_read,
2254                 .write_u64 = mem_cgroup_swappiness_write,
2255         },
2256 };
2257
2258 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2259 static struct cftype memsw_cgroup_files[] = {
2260         {
2261                 .name = "memsw.usage_in_bytes",
2262                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_USAGE),
2263                 .read_u64 = mem_cgroup_read,
2264         },
2265         {
2266                 .name = "memsw.max_usage_in_bytes",
2267                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_MAX_USAGE),
2268                 .trigger = mem_cgroup_reset,
2269                 .read_u64 = mem_cgroup_read,
2270         },
2271         {
2272                 .name = "memsw.limit_in_bytes",
2273                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_LIMIT),
2274                 .write_string = mem_cgroup_write,
2275                 .read_u64 = mem_cgroup_read,
2276         },
2277         {
2278                 .name = "memsw.failcnt",
2279                 .private = MEMFILE_PRIVATE(_MEMSWAP, RES_FAILCNT),
2280                 .trigger = mem_cgroup_reset,
2281                 .read_u64 = mem_cgroup_read,
2282         },
2283 };
2284
2285 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
2286 {
2287         if (!do_swap_account)
2288                 return 0;
2289         return cgroup_add_files(cont, ss, memsw_cgroup_files,
2290                                 ARRAY_SIZE(memsw_cgroup_files));
2291 };
2292 #else
2293 static int register_memsw_files(struct cgroup *cont, struct cgroup_subsys *ss)
2294 {
2295         return 0;
2296 }
2297 #endif
2298
2299 static int alloc_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
2300 {
2301         struct mem_cgroup_per_node *pn;
2302         struct mem_cgroup_per_zone *mz;
2303         enum lru_list l;
2304         int zone, tmp = node;
2305         /*
2306          * This routine is called against possible nodes.
2307          * But it's BUG to call kmalloc() against offline node.
2308          *
2309          * TODO: this routine can waste much memory for nodes which will
2310          *       never be onlined. It's better to use memory hotplug callback
2311          *       function.
2312          */
2313         if (!node_state(node, N_NORMAL_MEMORY))
2314                 tmp = -1;
2315         pn = kmalloc_node(sizeof(*pn), GFP_KERNEL, tmp);
2316         if (!pn)
2317                 return 1;
2318
2319         mem->info.nodeinfo[node] = pn;
2320         memset(pn, 0, sizeof(*pn));
2321
2322         for (zone = 0; zone < MAX_NR_ZONES; zone++) {
2323                 mz = &pn->zoneinfo[zone];
2324                 for_each_lru(l)
2325                         INIT_LIST_HEAD(&mz->lists[l]);
2326         }
2327         return 0;
2328 }
2329
2330 static void free_mem_cgroup_per_zone_info(struct mem_cgroup *mem, int node)
2331 {
2332         kfree(mem->info.nodeinfo[node]);
2333 }
2334
2335 static int mem_cgroup_size(void)
2336 {
2337         int cpustat_size = nr_cpu_ids * sizeof(struct mem_cgroup_stat_cpu);
2338         return sizeof(struct mem_cgroup) + cpustat_size;
2339 }
2340
2341 static struct mem_cgroup *mem_cgroup_alloc(void)
2342 {
2343         struct mem_cgroup *mem;
2344         int size = mem_cgroup_size();
2345
2346         if (size < PAGE_SIZE)
2347                 mem = kmalloc(size, GFP_KERNEL);
2348         else
2349                 mem = vmalloc(size);
2350
2351         if (mem)
2352                 memset(mem, 0, size);
2353         return mem;
2354 }
2355
2356 /*
2357  * At destroying mem_cgroup, references from swap_cgroup can remain.
2358  * (scanning all at force_empty is too costly...)
2359  *
2360  * Instead of clearing all references at force_empty, we remember
2361  * the number of reference from swap_cgroup and free mem_cgroup when
2362  * it goes down to 0.
2363  *
2364  * Removal of cgroup itself succeeds regardless of refs from swap.
2365  */
2366
2367 static void __mem_cgroup_free(struct mem_cgroup *mem)
2368 {
2369         int node;
2370
2371         free_css_id(&mem_cgroup_subsys, &mem->css);
2372
2373         for_each_node_state(node, N_POSSIBLE)
2374                 free_mem_cgroup_per_zone_info(mem, node);
2375
2376         if (mem_cgroup_size() < PAGE_SIZE)
2377                 kfree(mem);
2378         else
2379                 vfree(mem);
2380 }
2381
2382 static void mem_cgroup_get(struct mem_cgroup *mem)
2383 {
2384         atomic_inc(&mem->refcnt);
2385 }
2386
2387 static void mem_cgroup_put(struct mem_cgroup *mem)
2388 {
2389         if (atomic_dec_and_test(&mem->refcnt)) {
2390                 struct mem_cgroup *parent = parent_mem_cgroup(mem);
2391                 __mem_cgroup_free(mem);
2392                 if (parent)
2393                         mem_cgroup_put(parent);
2394         }
2395 }
2396
2397 /*
2398  * Returns the parent mem_cgroup in memcgroup hierarchy with hierarchy enabled.
2399  */
2400 static struct mem_cgroup *parent_mem_cgroup(struct mem_cgroup *mem)
2401 {
2402         if (!mem->res.parent)
2403                 return NULL;
2404         return mem_cgroup_from_res_counter(mem->res.parent, res);
2405 }
2406
2407 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2408 static void __init enable_swap_cgroup(void)
2409 {
2410         if (!mem_cgroup_disabled() && really_do_swap_account)
2411                 do_swap_account = 1;
2412 }
2413 #else
2414 static void __init enable_swap_cgroup(void)
2415 {
2416 }
2417 #endif
2418
2419 static struct cgroup_subsys_state * __ref
2420 mem_cgroup_create(struct cgroup_subsys *ss, struct cgroup *cont)
2421 {
2422         struct mem_cgroup *mem, *parent;
2423         long error = -ENOMEM;
2424         int node;
2425
2426         mem = mem_cgroup_alloc();
2427         if (!mem)
2428                 return ERR_PTR(error);
2429
2430         for_each_node_state(node, N_POSSIBLE)
2431                 if (alloc_mem_cgroup_per_zone_info(mem, node))
2432                         goto free_out;
2433         /* root ? */
2434         if (cont->parent == NULL) {
2435                 enable_swap_cgroup();
2436                 parent = NULL;
2437         } else {
2438                 parent = mem_cgroup_from_cont(cont->parent);
2439                 mem->use_hierarchy = parent->use_hierarchy;
2440         }
2441
2442         if (parent && parent->use_hierarchy) {
2443                 res_counter_init(&mem->res, &parent->res);
2444                 res_counter_init(&mem->memsw, &parent->memsw);
2445                 /*
2446                  * We increment refcnt of the parent to ensure that we can
2447                  * safely access it on res_counter_charge/uncharge.
2448                  * This refcnt will be decremented when freeing this
2449                  * mem_cgroup(see mem_cgroup_put).
2450                  */
2451                 mem_cgroup_get(parent);
2452         } else {
2453                 res_counter_init(&mem->res, NULL);
2454                 res_counter_init(&mem->memsw, NULL);
2455         }
2456         mem->last_scanned_child = 0;
2457         spin_lock_init(&mem->reclaim_param_lock);
2458
2459         if (parent)
2460                 mem->swappiness = get_swappiness(parent);
2461         atomic_set(&mem->refcnt, 1);
2462         return &mem->css;
2463 free_out:
2464         __mem_cgroup_free(mem);
2465         return ERR_PTR(error);
2466 }
2467
2468 static int mem_cgroup_pre_destroy(struct cgroup_subsys *ss,
2469                                         struct cgroup *cont)
2470 {
2471         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
2472
2473         return mem_cgroup_force_empty(mem, false);
2474 }
2475
2476 static void mem_cgroup_destroy(struct cgroup_subsys *ss,
2477                                 struct cgroup *cont)
2478 {
2479         struct mem_cgroup *mem = mem_cgroup_from_cont(cont);
2480
2481         mem_cgroup_put(mem);
2482 }
2483
2484 static int mem_cgroup_populate(struct cgroup_subsys *ss,
2485                                 struct cgroup *cont)
2486 {
2487         int ret;
2488
2489         ret = cgroup_add_files(cont, ss, mem_cgroup_files,
2490                                 ARRAY_SIZE(mem_cgroup_files));
2491
2492         if (!ret)
2493                 ret = register_memsw_files(cont, ss);
2494         return ret;
2495 }
2496
2497 static void mem_cgroup_move_task(struct cgroup_subsys *ss,
2498                                 struct cgroup *cont,
2499                                 struct cgroup *old_cont,
2500                                 struct task_struct *p)
2501 {
2502         mutex_lock(&memcg_tasklist);
2503         /*
2504          * FIXME: It's better to move charges of this process from old
2505          * memcg to new memcg. But it's just on TODO-List now.
2506          */
2507         mutex_unlock(&memcg_tasklist);
2508 }
2509
2510 struct cgroup_subsys mem_cgroup_subsys = {
2511         .name = "memory",
2512         .subsys_id = mem_cgroup_subsys_id,
2513         .create = mem_cgroup_create,
2514         .pre_destroy = mem_cgroup_pre_destroy,
2515         .destroy = mem_cgroup_destroy,
2516         .populate = mem_cgroup_populate,
2517         .attach = mem_cgroup_move_task,
2518         .early_init = 0,
2519         .use_id = 1,
2520 };
2521
2522 #ifdef CONFIG_CGROUP_MEM_RES_CTLR_SWAP
2523
2524 static int __init disable_swap_account(char *s)
2525 {
2526         really_do_swap_account = 0;
2527         return 1;
2528 }
2529 __setup("noswapaccount", disable_swap_account);
2530 #endif