memcg: make mem_cgroup_split_huge_fixup() more efficient
[linux-2.6.git] / mm / page_cgroup.c
index 5524e8b..f0559e0 100644 (file)
 #include <linux/swapops.h>
 #include <linux/kmemleak.h>
 
-static void __meminit init_page_cgroup(struct page_cgroup *pc, unsigned long id)
-{
-       pc->flags = 0;
-       set_page_cgroup_array_id(pc, id);
-       pc->mem_cgroup = NULL;
-       INIT_LIST_HEAD(&pc->lru);
-}
 static unsigned long total_usage;
 
 #if !defined(CONFIG_SPARSEMEM)
@@ -42,28 +35,13 @@ struct page_cgroup *lookup_page_cgroup(struct page *page)
        return base + offset;
 }
 
-struct page *lookup_cgroup_page(struct page_cgroup *pc)
-{
-       unsigned long pfn;
-       struct page *page;
-       pg_data_t *pgdat;
-
-       pgdat = NODE_DATA(page_cgroup_array_id(pc));
-       pfn = pc - pgdat->node_page_cgroup + pgdat->node_start_pfn;
-       page = pfn_to_page(pfn);
-       VM_BUG_ON(pc != lookup_page_cgroup(page));
-       return page;
-}
-
 static int __init alloc_node_page_cgroup(int nid)
 {
-       struct page_cgroup *base, *pc;
+       struct page_cgroup *base;
        unsigned long table_size;
-       unsigned long start_pfn, nr_pages, index;
+       unsigned long nr_pages;
 
-       start_pfn = NODE_DATA(nid)->node_start_pfn;
        nr_pages = NODE_DATA(nid)->node_spanned_pages;
-
        if (!nr_pages)
                return 0;
 
@@ -73,10 +51,6 @@ static int __init alloc_node_page_cgroup(int nid)
                        table_size, PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
        if (!base)
                return -ENOMEM;
-       for (index = 0; index < nr_pages; index++) {
-               pc = base + index;
-               init_page_cgroup(pc, nid);
-       }
        NODE_DATA(nid)->node_page_cgroup = base;
        total_usage += table_size;
        return 0;
@@ -117,31 +91,21 @@ struct page_cgroup *lookup_page_cgroup(struct page *page)
        return section->page_cgroup + pfn;
 }
 
-struct page *lookup_cgroup_page(struct page_cgroup *pc)
-{
-       struct mem_section *section;
-       struct page *page;
-       unsigned long nr;
-
-       nr = page_cgroup_array_id(pc);
-       section = __nr_to_section(nr);
-       page = pfn_to_page(pc - section->page_cgroup);
-       VM_BUG_ON(pc != lookup_page_cgroup(page));
-       return page;
-}
-
 static void *__meminit alloc_page_cgroup(size_t size, int nid)
 {
+       gfp_t flags = GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN;
        void *addr = NULL;
 
-       addr = alloc_pages_exact_nid(nid, size, GFP_KERNEL | __GFP_NOWARN);
-       if (addr)
+       addr = alloc_pages_exact_nid(nid, size, flags);
+       if (addr) {
+               kmemleak_alloc(addr, size, 1, flags);
                return addr;
+       }
 
        if (node_state(nid, N_HIGH_MEMORY))
-               addr = vmalloc_node(size, nid);
+               addr = vzalloc_node(size, nid);
        else
-               addr = vmalloc(size);
+               addr = vzalloc(size);
 
        return addr;
 }
@@ -162,21 +126,17 @@ static void free_page_cgroup(void *addr)
 }
 #endif
 
-static int __meminit init_section_page_cgroup(unsigned long pfn)
+static int __meminit init_section_page_cgroup(unsigned long pfn, int nid)
 {
-       struct page_cgroup *base, *pc;
        struct mem_section *section;
+       struct page_cgroup *base;
        unsigned long table_size;
-       unsigned long nr;
-       int nid, index;
 
-       nr = pfn_to_section_nr(pfn);
-       section = __nr_to_section(nr);
+       section = __pfn_to_section(pfn);
 
        if (section->page_cgroup)
                return 0;
 
-       nid = page_to_nid(pfn_to_page(pfn));
        table_size = sizeof(struct page_cgroup) * PAGES_PER_SECTION;
        base = alloc_page_cgroup(table_size, nid);
 
@@ -192,11 +152,11 @@ static int __meminit init_section_page_cgroup(unsigned long pfn)
                return -ENOMEM;
        }
 
-       for (index = 0; index < PAGES_PER_SECTION; index++) {
-               pc = base + index;
-               init_page_cgroup(pc, nr);
-       }
-
+       /*
+        * The passed "pfn" may not be aligned to SECTION.  For the calculation
+        * we need to apply a mask.
+        */
+       pfn &= PAGE_SECTION_MASK;
        section->page_cgroup = base - pfn;
        total_usage += table_size;
        return 0;
@@ -222,13 +182,23 @@ int __meminit online_page_cgroup(unsigned long start_pfn,
        unsigned long start, end, pfn;
        int fail = 0;
 
-       start = start_pfn & ~(PAGES_PER_SECTION - 1);
-       end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
+       start = SECTION_ALIGN_DOWN(start_pfn);
+       end = SECTION_ALIGN_UP(start_pfn + nr_pages);
+
+       if (nid == -1) {
+               /*
+                * In this case, "nid" already exists and contains valid memory.
+                * "start_pfn" passed to us is a pfn which is an arg for
+                * online__pages(), and start_pfn should exist.
+                */
+               nid = pfn_to_nid(start_pfn);
+               VM_BUG_ON(!node_state(nid, N_ONLINE));
+       }
 
        for (pfn = start; !fail && pfn < end; pfn += PAGES_PER_SECTION) {
                if (!pfn_present(pfn))
                        continue;
-               fail = init_section_page_cgroup(pfn);
+               fail = init_section_page_cgroup(pfn, nid);
        }
        if (!fail)
                return 0;
@@ -245,8 +215,8 @@ int __meminit offline_page_cgroup(unsigned long start_pfn,
 {
        unsigned long start, end, pfn;
 
-       start = start_pfn & ~(PAGES_PER_SECTION - 1);
-       end = ALIGN(start_pfn + nr_pages, PAGES_PER_SECTION);
+       start = SECTION_ALIGN_DOWN(start_pfn);
+       end = SECTION_ALIGN_UP(start_pfn + nr_pages);
 
        for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION)
                __free_page_cgroup(pfn);
@@ -284,25 +254,47 @@ static int __meminit page_cgroup_callback(struct notifier_block *self,
 void __init page_cgroup_init(void)
 {
        unsigned long pfn;
-       int fail = 0;
+       int nid;
 
        if (mem_cgroup_disabled())
                return;
 
-       for (pfn = 0; !fail && pfn < max_pfn; pfn += PAGES_PER_SECTION) {
-               if (!pfn_present(pfn))
-                       continue;
-               fail = init_section_page_cgroup(pfn);
-       }
-       if (fail) {
-               printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
-               panic("Out of memory");
-       } else {
-               hotplug_memory_notifier(page_cgroup_callback, 0);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               unsigned long start_pfn, end_pfn;
+
+               start_pfn = node_start_pfn(nid);
+               end_pfn = node_end_pfn(nid);
+               /*
+                * start_pfn and end_pfn may not be aligned to SECTION and the
+                * page->flags of out of node pages are not initialized.  So we
+                * scan [start_pfn, the biggest section's pfn < end_pfn) here.
+                */
+               for (pfn = start_pfn;
+                    pfn < end_pfn;
+                     pfn = ALIGN(pfn + 1, PAGES_PER_SECTION)) {
+
+                       if (!pfn_valid(pfn))
+                               continue;
+                       /*
+                        * Nodes's pfns can be overlapping.
+                        * We know some arch can have a nodes layout such as
+                        * -------------pfn-------------->
+                        * N0 | N1 | N2 | N0 | N1 | N2|....
+                        */
+                       if (pfn_to_nid(pfn) != nid)
+                               continue;
+                       if (init_section_page_cgroup(pfn, nid))
+                               goto oom;
+               }
        }
+       hotplug_memory_notifier(page_cgroup_callback, 0);
        printk(KERN_INFO "allocated %ld bytes of page_cgroup\n", total_usage);
-       printk(KERN_INFO "please try 'cgroup_disable=memory' option if you don't"
-       " want memory cgroups\n");
+       printk(KERN_INFO "please try 'cgroup_disable=memory' option if you "
+                        "don't want memory cgroups\n");
+       return;
+oom:
+       printk(KERN_CRIT "try 'cgroup_disable=memory' boot option\n");
+       panic("Out of memory");
 }
 
 void __meminit pgdat_page_cgroup_init(struct pglist_data *pgdat)
@@ -322,7 +314,7 @@ struct swap_cgroup_ctrl {
        spinlock_t      lock;
 };
 
-struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
+static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
 
 struct swap_cgroup {
        unsigned short          id;
@@ -478,11 +470,10 @@ int swap_cgroup_swapon(int type, unsigned long max_pages)
        length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
        array_size = length * sizeof(void *);
 
-       array = vmalloc(array_size);
+       array = vzalloc(array_size);
        if (!array)
                goto nomem;
 
-       memset(array, 0, array_size);
        ctrl = &swap_cgroup_ctrl[type];
        mutex_lock(&swap_cgroup_mutex);
        ctrl->length = length;
@@ -492,8 +483,8 @@ int swap_cgroup_swapon(int type, unsigned long max_pages)
                /* memory shortage */
                ctrl->map = NULL;
                ctrl->length = 0;
-               vfree(array);
                mutex_unlock(&swap_cgroup_mutex);
+               vfree(array);
                goto nomem;
        }
        mutex_unlock(&swap_cgroup_mutex);
@@ -502,13 +493,14 @@ int swap_cgroup_swapon(int type, unsigned long max_pages)
 nomem:
        printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
        printk(KERN_INFO
-               "swap_cgroup can be disabled by noswapaccount boot option\n");
+               "swap_cgroup can be disabled by swapaccount=0 boot option\n");
        return -ENOMEM;
 }
 
 void swap_cgroup_swapoff(int type)
 {
-       int i;
+       struct page **map;
+       unsigned long i, length;
        struct swap_cgroup_ctrl *ctrl;
 
        if (!do_swap_account)
@@ -516,17 +508,20 @@ void swap_cgroup_swapoff(int type)
 
        mutex_lock(&swap_cgroup_mutex);
        ctrl = &swap_cgroup_ctrl[type];
-       if (ctrl->map) {
-               for (i = 0; i < ctrl->length; i++) {
-                       struct page *page = ctrl->map[i];
+       map = ctrl->map;
+       length = ctrl->length;
+       ctrl->map = NULL;
+       ctrl->length = 0;
+       mutex_unlock(&swap_cgroup_mutex);
+
+       if (map) {
+               for (i = 0; i < length; i++) {
+                       struct page *page = map[i];
                        if (page)
                                __free_page(page);
                }
-               vfree(ctrl->map);
-               ctrl->map = NULL;
-               ctrl->length = 0;
+               vfree(map);
        }
-       mutex_unlock(&swap_cgroup_mutex);
 }
 
 #endif