memcg: allocate all page_cgroup at boot
[linux-3.10.git] / mm / page_cgroup.c
1 #include <linux/mm.h>
2 #include <linux/mmzone.h>
3 #include <linux/bootmem.h>
4 #include <linux/bit_spinlock.h>
5 #include <linux/page_cgroup.h>
6 #include <linux/hash.h>
7 #include <linux/memory.h>
8
9 static void __meminit
10 __init_page_cgroup(struct page_cgroup *pc, unsigned long pfn)
11 {
12         pc->flags = 0;
13         pc->mem_cgroup = NULL;
14         pc->page = pfn_to_page(pfn);
15 }
16 static unsigned long total_usage;
17
18 #if !defined(CONFIG_SPARSEMEM)
19
20
21 void __init pgdat_page_cgroup_init(struct pglist_data *pgdat)
22 {
23         pgdat->node_page_cgroup = NULL;
24 }
25
26 struct page_cgroup *lookup_page_cgroup(struct page *page)
27 {
28         unsigned long pfn = page_to_pfn(page);
29         unsigned long offset;
30         struct page_cgroup *base;
31
32         base = NODE_DATA(page_to_nid(page))->node_page_cgroup;
33         if (unlikely(!base))
34                 return NULL;
35
36         offset = pfn - NODE_DATA(page_to_nid(page))->node_start_pfn;
37         return base + offset;
38 }
39
40 static int __init alloc_node_page_cgroup(int nid)
41 {
42         struct page_cgroup *base, *pc;
43         unsigned long table_size;
44         unsigned long start_pfn, nr_pages, index;
45
46         start_pfn = NODE_DATA(nid)->node_start_pfn;
47         nr_pages = NODE_DATA(nid)->node_spanned_pages;
48
49         table_size = sizeof(struct page_cgroup) * nr_pages;
50
51         base = __alloc_bootmem_node_nopanic(NODE_DATA(nid),
52                         table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
53         if (!base)
54                 return -ENOMEM;
55         for (index = 0; index < nr_pages; index++) {
56                 pc = base + index;
57                 __init_page_cgroup(pc, start_pfn + index);
58         }
59         NODE_DATA(nid)->node_page_cgroup = base;
60         total_usage += table_size;
61         return 0;
62 }
63
64 void __init page_cgroup_init(void)
65 {
66
67         int nid, fail;
68
69         for_each_online_node(nid)  {
70                 fail = alloc_node_page_cgroup(nid);
71                 if (fail)
72                         goto fail;
73         }
74         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
75         printk(KERN_INFO "please try cgroup_disable=memory option if you"
76         " don't want\n");
77         return;
78 fail:
79         printk(KERN_CRIT "allocation of page_cgroup was failed.\n");
80         printk(KERN_CRIT "please try cgroup_disable=memory boot option\n");
81         panic("Out of memory");
82 }
83
84 #else /* CONFIG_FLAT_NODE_MEM_MAP */
85
86 struct page_cgroup *lookup_page_cgroup(struct page *page)
87 {
88         unsigned long pfn = page_to_pfn(page);
89         struct mem_section *section = __pfn_to_section(pfn);
90
91         return section->page_cgroup + pfn;
92 }
93
94 int __meminit init_section_page_cgroup(unsigned long pfn)
95 {
96         struct mem_section *section;
97         struct page_cgroup *base, *pc;
98         unsigned long table_size;
99         int nid, index;
100
101         section = __pfn_to_section(pfn);
102
103         if (section->page_cgroup)
104                 return 0;
105
106         nid = page_to_nid(pfn_to_page(pfn));
107
108         table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
109         base = kmalloc_node(table_size, GFP_KERNEL, nid);
110         if (!base)
111                 base = vmalloc_node(table_size, nid);
112
113         if (!base) {
114                 printk(KERN_ERR "page cgroup allocation failure\n");
115                 return -ENOMEM;
116         }
117
118         for (index = 0; index < PAGES_PER_SECTION; index++) {
119                 pc = base + index;
120                 __init_page_cgroup(pc, pfn + index);
121         }
122
123         section = __pfn_to_section(pfn);
124         section->page_cgroup = base - pfn;
125         total_usage += table_size;
126         return 0;
127 }
128 #ifdef CONFIG_MEMORY_HOTPLUG
129 void __free_page_cgroup(unsigned long pfn)
130 {
131         struct mem_section *ms;
132         struct page_cgroup *base;
133
134         ms = __pfn_to_section(pfn);
135         if (!ms || !ms->page_cgroup)
136                 return;
137         base = ms->page_cgroup + pfn;
138         ms->page_cgroup = NULL;
139         if (is_vmalloc_addr(base))
140                 vfree(base);
141         else
142                 kfree(base);
143 }
144
145 int online_page_cgroup(unsigned long start_pfn,
146                         unsigned long nr_pages,
147                         int nid)
148 {
149         unsigned long start, end, pfn;
150         int fail = 0;
151
152         start = start_pfn & (PAGES_PER_SECTION - 1);
153         end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
154
155         for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
156                 if (!pfn_present(pfn))
157                         continue;
158                 fail = init_section_page_cgroup(pfn);
159         }
160         if (!fail)
161                 return 0;
162
163         /* rollback */
164         for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
165                 __free_page_cgroup(pfn);
166
167         return -ENOMEM;
168 }
169
170 int offline_page_cgroup(unsigned long start_pfn,
171                 unsigned long nr_pages, int nid)
172 {
173         unsigned long start, end, pfn;
174
175         start = start_pfn & (PAGES_PER_SECTION - 1);
176         end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
177
178         for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
179                 __free_page_cgroup(pfn);
180         return 0;
181
182 }
183
184 static int page_cgroup_callback(struct notifier_block *self,
185                                unsigned long action, void *arg)
186 {
187         struct memory_notify *mn = arg;
188         int ret = 0;
189         switch (action) {
190         case MEM_GOING_ONLINE:
191                 ret = online_page_cgroup(mn->start_pfn,
192                                    mn->nr_pages, mn->status_change_nid);
193                 break;
194         case MEM_CANCEL_ONLINE:
195         case MEM_OFFLINE:
196                 offline_page_cgroup(mn->start_pfn,
197                                 mn->nr_pages, mn->status_change_nid);
198                 break;
199         case MEM_GOING_OFFLINE:
200                 break;
201         case MEM_ONLINE:
202         case MEM_CANCEL_OFFLINE:
203                 break;
204         }
205         ret = notifier_from_errno(ret);
206         return ret;
207 }
208
209 #endif
210
211 void __init page_cgroup_init(void)
212 {
213         unsigned long pfn;
214         int fail = 0;
215
216         for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
217                 if (!pfn_present(pfn))
218                         continue;
219                 fail = init_section_page_cgroup(pfn);
220         }
221         if (fail) {
222                 printk(KERN_CRIT "try cgroup_disable=memory boot option\n");
223                 panic("Out of memory");
224         } else {
225                 hotplug_memory_notifier(page_cgroup_callback, 0);
226         }
227         printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
228         printk(KERN_INFO "please try cgroup_disable=memory option if you don't"
229         " want\n");
230 }
231
232 void __init pgdat_page_cgroup_init(struct pglist_data *pgdat)
233 {
234         return;
235 }
236
237 #endif