memcg: add interface to move charge at task migration
[linux-2.6.git] / mm / hugetlb.c
index 324d1ab..3a5aeb3 100644 (file)
@@ -24,6 +24,7 @@
 #include <asm/io.h>
 
 #include <linux/hugetlb.h>
+#include <linux/node.h>
 #include "internal.h"
 
 const unsigned long hugetlb_zero = 0, hugetlb_infinity = ~0UL;
@@ -401,7 +402,7 @@ static void clear_huge_page(struct page *page,
 {
        int i;
 
-       if (unlikely(sz > MAX_ORDER_NR_PAGES)) {
+       if (unlikely(sz/PAGE_SIZE > MAX_ORDER_NR_PAGES)) {
                clear_gigantic_page(page, addr, sz);
                return;
        }
@@ -941,14 +942,14 @@ static void return_unused_surplus_pages(struct hstate *h,
 
        /*
         * We want to release as many surplus pages as possible, spread
-        * evenly across all nodes. Iterate across all nodes until we
-        * can no longer free unreserved surplus pages. This occurs when
-        * the nodes with surplus pages have no free pages.
-        * free_pool_huge_page() will balance the the frees across the
-        * on-line nodes for us and will handle the hstate accounting.
+        * evenly across all nodes with memory. Iterate across these nodes
+        * until we can no longer free unreserved surplus pages. This occurs
+        * when the nodes with surplus pages have no free pages.
+        * free_pool_huge_page() will balance the the freed pages across the
+        * on-line nodes with memory and will handle the hstate accounting.
         */
        while (nr_pages--) {
-               if (!free_pool_huge_page(h, &node_online_map, 1))
+               if (!free_pool_huge_page(h, &node_states[N_HIGH_MEMORY], 1))
                        break;
        }
 }
@@ -1052,14 +1053,14 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma,
 int __weak alloc_bootmem_huge_page(struct hstate *h)
 {
        struct huge_bootmem_page *m;
-       int nr_nodes = nodes_weight(node_online_map);
+       int nr_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
 
        while (nr_nodes) {
                void *addr;
 
                addr = __alloc_bootmem_node_nopanic(
                                NODE_DATA(hstate_next_node_to_alloc(h,
-                                                       &node_online_map)),
+                                               &node_states[N_HIGH_MEMORY])),
                                huge_page_size(h), huge_page_size(h), 0);
 
                if (addr) {
@@ -1114,7 +1115,8 @@ static void __init hugetlb_hstate_alloc_pages(struct hstate *h)
                if (h->order >= MAX_ORDER) {
                        if (!alloc_bootmem_huge_page(h))
                                break;
-               } else if (!alloc_fresh_huge_page(h, &node_online_map))
+               } else if (!alloc_fresh_huge_page(h,
+                                        &node_states[N_HIGH_MEMORY]))
                        break;
        }
        h->max_huge_pages = i;
@@ -1276,6 +1278,9 @@ static unsigned long set_max_huge_pages(struct hstate *h, unsigned long count,
                if (!ret)
                        goto out;
 
+               /* Bail for signals. Probably ctrl-c from user */
+               if (signal_pending(current))
+                       goto out;
        }
 
        /*
@@ -1320,43 +1325,117 @@ out:
 static struct kobject *hugepages_kobj;
 static struct kobject *hstate_kobjs[HUGE_MAX_HSTATE];
 
-static struct hstate *kobj_to_hstate(struct kobject *kobj)
+static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp);
+
+static struct hstate *kobj_to_hstate(struct kobject *kobj, int *nidp)
 {
        int i;
+
        for (i = 0; i < HUGE_MAX_HSTATE; i++)
-               if (hstate_kobjs[i] == kobj)
+               if (hstate_kobjs[i] == kobj) {
+                       if (nidp)
+                               *nidp = NUMA_NO_NODE;
                        return &hstates[i];
-       BUG();
-       return NULL;
+               }
+
+       return kobj_to_node_hstate(kobj, nidp);
 }
 
-static ssize_t nr_hugepages_show(struct kobject *kobj,
+static ssize_t nr_hugepages_show_common(struct kobject *kobj,
                                        struct kobj_attribute *attr, char *buf)
 {
-       struct hstate *h = kobj_to_hstate(kobj);
-       return sprintf(buf, "%lu\n", h->nr_huge_pages);
+       struct hstate *h;
+       unsigned long nr_huge_pages;
+       int nid;
+
+       h = kobj_to_hstate(kobj, &nid);
+       if (nid == NUMA_NO_NODE)
+               nr_huge_pages = h->nr_huge_pages;
+       else
+               nr_huge_pages = h->nr_huge_pages_node[nid];
+
+       return sprintf(buf, "%lu\n", nr_huge_pages);
 }
-static ssize_t nr_hugepages_store(struct kobject *kobj,
-               struct kobj_attribute *attr, const char *buf, size_t count)
+static ssize_t nr_hugepages_store_common(bool obey_mempolicy,
+                       struct kobject *kobj, struct kobj_attribute *attr,
+                       const char *buf, size_t len)
 {
        int err;
-       unsigned long input;
-       struct hstate *h = kobj_to_hstate(kobj);
+       int nid;
+       unsigned long count;
+       struct hstate *h;
+       NODEMASK_ALLOC(nodemask_t, nodes_allowed, GFP_KERNEL | __GFP_NORETRY);
 
-       err = strict_strtoul(buf, 10, &input);
+       err = strict_strtoul(buf, 10, &count);
        if (err)
                return 0;
 
-       h->max_huge_pages = set_max_huge_pages(h, input, &node_online_map);
+       h = kobj_to_hstate(kobj, &nid);
+       if (nid == NUMA_NO_NODE) {
+               /*
+                * global hstate attribute
+                */
+               if (!(obey_mempolicy &&
+                               init_nodemask_of_mempolicy(nodes_allowed))) {
+                       NODEMASK_FREE(nodes_allowed);
+                       nodes_allowed = &node_states[N_HIGH_MEMORY];
+               }
+       } else if (nodes_allowed) {
+               /*
+                * per node hstate attribute: adjust count to global,
+                * but restrict alloc/free to the specified node.
+                */
+               count += h->nr_huge_pages - h->nr_huge_pages_node[nid];
+               init_nodemask_of_node(nodes_allowed, nid);
+       } else
+               nodes_allowed = &node_states[N_HIGH_MEMORY];
+
+       h->max_huge_pages = set_max_huge_pages(h, count, nodes_allowed);
 
-       return count;
+       if (nodes_allowed != &node_states[N_HIGH_MEMORY])
+               NODEMASK_FREE(nodes_allowed);
+
+       return len;
+}
+
+static ssize_t nr_hugepages_show(struct kobject *kobj,
+                                      struct kobj_attribute *attr, char *buf)
+{
+       return nr_hugepages_show_common(kobj, attr, buf);
+}
+
+static ssize_t nr_hugepages_store(struct kobject *kobj,
+              struct kobj_attribute *attr, const char *buf, size_t len)
+{
+       return nr_hugepages_store_common(false, kobj, attr, buf, len);
 }
 HSTATE_ATTR(nr_hugepages);
 
+#ifdef CONFIG_NUMA
+
+/*
+ * hstate attribute for optionally mempolicy-based constraint on persistent
+ * huge page alloc/free.
+ */
+static ssize_t nr_hugepages_mempolicy_show(struct kobject *kobj,
+                                      struct kobj_attribute *attr, char *buf)
+{
+       return nr_hugepages_show_common(kobj, attr, buf);
+}
+
+static ssize_t nr_hugepages_mempolicy_store(struct kobject *kobj,
+              struct kobj_attribute *attr, const char *buf, size_t len)
+{
+       return nr_hugepages_store_common(true, kobj, attr, buf, len);
+}
+HSTATE_ATTR(nr_hugepages_mempolicy);
+#endif
+
+
 static ssize_t nr_overcommit_hugepages_show(struct kobject *kobj,
                                        struct kobj_attribute *attr, char *buf)
 {
-       struct hstate *h = kobj_to_hstate(kobj);
+       struct hstate *h = kobj_to_hstate(kobj, NULL);
        return sprintf(buf, "%lu\n", h->nr_overcommit_huge_pages);
 }
 static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
@@ -1364,7 +1443,7 @@ static ssize_t nr_overcommit_hugepages_store(struct kobject *kobj,
 {
        int err;
        unsigned long input;
-       struct hstate *h = kobj_to_hstate(kobj);
+       struct hstate *h = kobj_to_hstate(kobj, NULL);
 
        err = strict_strtoul(buf, 10, &input);
        if (err)
@@ -1381,15 +1460,24 @@ HSTATE_ATTR(nr_overcommit_hugepages);
 static ssize_t free_hugepages_show(struct kobject *kobj,
                                        struct kobj_attribute *attr, char *buf)
 {
-       struct hstate *h = kobj_to_hstate(kobj);
-       return sprintf(buf, "%lu\n", h->free_huge_pages);
+       struct hstate *h;
+       unsigned long free_huge_pages;
+       int nid;
+
+       h = kobj_to_hstate(kobj, &nid);
+       if (nid == NUMA_NO_NODE)
+               free_huge_pages = h->free_huge_pages;
+       else
+               free_huge_pages = h->free_huge_pages_node[nid];
+
+       return sprintf(buf, "%lu\n", free_huge_pages);
 }
 HSTATE_ATTR_RO(free_hugepages);
 
 static ssize_t resv_hugepages_show(struct kobject *kobj,
                                        struct kobj_attribute *attr, char *buf)
 {
-       struct hstate *h = kobj_to_hstate(kobj);
+       struct hstate *h = kobj_to_hstate(kobj, NULL);
        return sprintf(buf, "%lu\n", h->resv_huge_pages);
 }
 HSTATE_ATTR_RO(resv_hugepages);
@@ -1397,8 +1485,17 @@ HSTATE_ATTR_RO(resv_hugepages);
 static ssize_t surplus_hugepages_show(struct kobject *kobj,
                                        struct kobj_attribute *attr, char *buf)
 {
-       struct hstate *h = kobj_to_hstate(kobj);
-       return sprintf(buf, "%lu\n", h->surplus_huge_pages);
+       struct hstate *h;
+       unsigned long surplus_huge_pages;
+       int nid;
+
+       h = kobj_to_hstate(kobj, &nid);
+       if (nid == NUMA_NO_NODE)
+               surplus_huge_pages = h->surplus_huge_pages;
+       else
+               surplus_huge_pages = h->surplus_huge_pages_node[nid];
+
+       return sprintf(buf, "%lu\n", surplus_huge_pages);
 }
 HSTATE_ATTR_RO(surplus_hugepages);
 
@@ -1408,6 +1505,9 @@ static struct attribute *hstate_attrs[] = {
        &free_hugepages_attr.attr,
        &resv_hugepages_attr.attr,
        &surplus_hugepages_attr.attr,
+#ifdef CONFIG_NUMA
+       &nr_hugepages_mempolicy_attr.attr,
+#endif
        NULL,
 };
 
@@ -1415,19 +1515,20 @@ static struct attribute_group hstate_attr_group = {
        .attrs = hstate_attrs,
 };
 
-static int __init hugetlb_sysfs_add_hstate(struct hstate *h)
+static int hugetlb_sysfs_add_hstate(struct hstate *h, struct kobject *parent,
+                                   struct kobject **hstate_kobjs,
+                                   struct attribute_group *hstate_attr_group)
 {
        int retval;
+       int hi = h - hstates;
 
-       hstate_kobjs[h - hstates] = kobject_create_and_add(h->name,
-                                                       hugepages_kobj);
-       if (!hstate_kobjs[h - hstates])
+       hstate_kobjs[hi] = kobject_create_and_add(h->name, parent);
+       if (!hstate_kobjs[hi])
                return -ENOMEM;
 
-       retval = sysfs_create_group(hstate_kobjs[h - hstates],
-                                                       &hstate_attr_group);
+       retval = sysfs_create_group(hstate_kobjs[hi], hstate_attr_group);
        if (retval)
-               kobject_put(hstate_kobjs[h - hstates]);
+               kobject_put(hstate_kobjs[hi]);
 
        return retval;
 }
@@ -1442,17 +1543,184 @@ static void __init hugetlb_sysfs_init(void)
                return;
 
        for_each_hstate(h) {
-               err = hugetlb_sysfs_add_hstate(h);
+               err = hugetlb_sysfs_add_hstate(h, hugepages_kobj,
+                                        hstate_kobjs, &hstate_attr_group);
                if (err)
                        printk(KERN_ERR "Hugetlb: Unable to add hstate %s",
                                                                h->name);
        }
 }
 
+#ifdef CONFIG_NUMA
+
+/*
+ * node_hstate/s - associate per node hstate attributes, via their kobjects,
+ * with node sysdevs in node_devices[] using a parallel array.  The array
+ * index of a node sysdev or _hstate == node id.
+ * This is here to avoid any static dependency of the node sysdev driver, in
+ * the base kernel, on the hugetlb module.
+ */
+struct node_hstate {
+       struct kobject          *hugepages_kobj;
+       struct kobject          *hstate_kobjs[HUGE_MAX_HSTATE];
+};
+struct node_hstate node_hstates[MAX_NUMNODES];
+
+/*
+ * A subset of global hstate attributes for node sysdevs
+ */
+static struct attribute *per_node_hstate_attrs[] = {
+       &nr_hugepages_attr.attr,
+       &free_hugepages_attr.attr,
+       &surplus_hugepages_attr.attr,
+       NULL,
+};
+
+static struct attribute_group per_node_hstate_attr_group = {
+       .attrs = per_node_hstate_attrs,
+};
+
+/*
+ * kobj_to_node_hstate - lookup global hstate for node sysdev hstate attr kobj.
+ * Returns node id via non-NULL nidp.
+ */
+static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
+{
+       int nid;
+
+       for (nid = 0; nid < nr_node_ids; nid++) {
+               struct node_hstate *nhs = &node_hstates[nid];
+               int i;
+               for (i = 0; i < HUGE_MAX_HSTATE; i++)
+                       if (nhs->hstate_kobjs[i] == kobj) {
+                               if (nidp)
+                                       *nidp = nid;
+                               return &hstates[i];
+                       }
+       }
+
+       BUG();
+       return NULL;
+}
+
+/*
+ * Unregister hstate attributes from a single node sysdev.
+ * No-op if no hstate attributes attached.
+ */
+void hugetlb_unregister_node(struct node *node)
+{
+       struct hstate *h;
+       struct node_hstate *nhs = &node_hstates[node->sysdev.id];
+
+       if (!nhs->hugepages_kobj)
+               return;         /* no hstate attributes */
+
+       for_each_hstate(h)
+               if (nhs->hstate_kobjs[h - hstates]) {
+                       kobject_put(nhs->hstate_kobjs[h - hstates]);
+                       nhs->hstate_kobjs[h - hstates] = NULL;
+               }
+
+       kobject_put(nhs->hugepages_kobj);
+       nhs->hugepages_kobj = NULL;
+}
+
+/*
+ * hugetlb module exit:  unregister hstate attributes from node sysdevs
+ * that have them.
+ */
+static void hugetlb_unregister_all_nodes(void)
+{
+       int nid;
+
+       /*
+        * disable node sysdev registrations.
+        */
+       register_hugetlbfs_with_node(NULL, NULL);
+
+       /*
+        * remove hstate attributes from any nodes that have them.
+        */
+       for (nid = 0; nid < nr_node_ids; nid++)
+               hugetlb_unregister_node(&node_devices[nid]);
+}
+
+/*
+ * Register hstate attributes for a single node sysdev.
+ * No-op if attributes already registered.
+ */
+void hugetlb_register_node(struct node *node)
+{
+       struct hstate *h;
+       struct node_hstate *nhs = &node_hstates[node->sysdev.id];
+       int err;
+
+       if (nhs->hugepages_kobj)
+               return;         /* already allocated */
+
+       nhs->hugepages_kobj = kobject_create_and_add("hugepages",
+                                                       &node->sysdev.kobj);
+       if (!nhs->hugepages_kobj)
+               return;
+
+       for_each_hstate(h) {
+               err = hugetlb_sysfs_add_hstate(h, nhs->hugepages_kobj,
+                                               nhs->hstate_kobjs,
+                                               &per_node_hstate_attr_group);
+               if (err) {
+                       printk(KERN_ERR "Hugetlb: Unable to add hstate %s"
+                                       " for node %d\n",
+                                               h->name, node->sysdev.id);
+                       hugetlb_unregister_node(node);
+                       break;
+               }
+       }
+}
+
+/*
+ * hugetlb init time:  register hstate attributes for all registered node
+ * sysdevs of nodes that have memory.  All on-line nodes should have
+ * registered their associated sysdev by this time.
+ */
+static void hugetlb_register_all_nodes(void)
+{
+       int nid;
+
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               struct node *node = &node_devices[nid];
+               if (node->sysdev.id == nid)
+                       hugetlb_register_node(node);
+       }
+
+       /*
+        * Let the node sysdev driver know we're here so it can
+        * [un]register hstate attributes on node hotplug.
+        */
+       register_hugetlbfs_with_node(hugetlb_register_node,
+                                    hugetlb_unregister_node);
+}
+#else  /* !CONFIG_NUMA */
+
+static struct hstate *kobj_to_node_hstate(struct kobject *kobj, int *nidp)
+{
+       BUG();
+       if (nidp)
+               *nidp = -1;
+       return NULL;
+}
+
+static void hugetlb_unregister_all_nodes(void) { }
+
+static void hugetlb_register_all_nodes(void) { }
+
+#endif
+
 static void __exit hugetlb_exit(void)
 {
        struct hstate *h;
 
+       hugetlb_unregister_all_nodes();
+
        for_each_hstate(h) {
                kobject_put(hstate_kobjs[h - hstates]);
        }
@@ -1487,6 +1755,8 @@ static int __init hugetlb_init(void)
 
        hugetlb_sysfs_init();
 
+       hugetlb_register_all_nodes();
+
        return 0;
 }
 module_init(hugetlb_init);
@@ -1510,8 +1780,8 @@ void __init hugetlb_add_hstate(unsigned order)
        h->free_huge_pages = 0;
        for (i = 0; i < MAX_NUMNODES; ++i)
                INIT_LIST_HEAD(&h->hugepage_freelists[i]);
-       h->next_nid_to_alloc = first_node(node_online_map);
-       h->next_nid_to_free = first_node(node_online_map);
+       h->next_nid_to_alloc = first_node(node_states[N_HIGH_MEMORY]);
+       h->next_nid_to_free = first_node(node_states[N_HIGH_MEMORY]);
        snprintf(h->name, HSTATE_NAME_LEN, "hugepages-%lukB",
                                        huge_page_size(h)/1024);
 
@@ -1574,9 +1844,9 @@ static unsigned int cpuset_mems_nr(unsigned int *array)
 }
 
 #ifdef CONFIG_SYSCTL
-int hugetlb_sysctl_handler(struct ctl_table *table, int write,
-                          void __user *buffer,
-                          size_t *length, loff_t *ppos)
+static int hugetlb_sysctl_handler_common(bool obey_mempolicy,
+                        struct ctl_table *table, int write,
+                        void __user *buffer, size_t *length, loff_t *ppos)
 {
        struct hstate *h = &default_hstate;
        unsigned long tmp;
@@ -1588,13 +1858,40 @@ int hugetlb_sysctl_handler(struct ctl_table *table, int write,
        table->maxlen = sizeof(unsigned long);
        proc_doulongvec_minmax(table, write, buffer, length, ppos);
 
-       if (write)
-               h->max_huge_pages = set_max_huge_pages(h, tmp,
-                                                       &node_online_map);
+       if (write) {
+               NODEMASK_ALLOC(nodemask_t, nodes_allowed,
+                                               GFP_KERNEL | __GFP_NORETRY);
+               if (!(obey_mempolicy &&
+                              init_nodemask_of_mempolicy(nodes_allowed))) {
+                       NODEMASK_FREE(nodes_allowed);
+                       nodes_allowed = &node_states[N_HIGH_MEMORY];
+               }
+               h->max_huge_pages = set_max_huge_pages(h, tmp, nodes_allowed);
+
+               if (nodes_allowed != &node_states[N_HIGH_MEMORY])
+                       NODEMASK_FREE(nodes_allowed);
+       }
 
        return 0;
 }
 
+int hugetlb_sysctl_handler(struct ctl_table *table, int write,
+                         void __user *buffer, size_t *length, loff_t *ppos)
+{
+
+       return hugetlb_sysctl_handler_common(false, table, write,
+                                                       buffer, length, ppos);
+}
+
+#ifdef CONFIG_NUMA
+int hugetlb_mempolicy_sysctl_handler(struct ctl_table *table, int write,
+                         void __user *buffer, size_t *length, loff_t *ppos)
+{
+       return hugetlb_sysctl_handler_common(true, table, write,
+                                                       buffer, length, ppos);
+}
+#endif /* CONFIG_NUMA */
+
 int hugetlb_treat_movable_handler(struct ctl_table *table, int write,
                        void __user *buffer,
                        size_t *length, loff_t *ppos)
@@ -1790,7 +2087,7 @@ static void set_huge_ptep_writable(struct vm_area_struct *vma,
 
        entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
        if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1)) {
-               update_mmu_cache(vma, address, entry);
+               update_mmu_cache(vma, address, ptep);
        }
 }
 
@@ -1942,6 +2239,12 @@ static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
                + (vma->vm_pgoff >> PAGE_SHIFT);
        mapping = (struct address_space *)page_private(page);
 
+       /*
+        * Take the mapping lock for the duration of the table walk. As
+        * this mapping should be shared between all the VMAs,
+        * __unmap_hugepage_range() is called as the lock is already held
+        */
+       spin_lock(&mapping->i_mmap_lock);
        vma_prio_tree_foreach(iter_vma, &iter, &mapping->i_mmap, pgoff, pgoff) {
                /* Do not unmap the current VMA */
                if (iter_vma == vma)
@@ -1955,10 +2258,11 @@ static int unmap_ref_private(struct mm_struct *mm, struct vm_area_struct *vma,
                 * from the time of fork. This would look like data corruption
                 */
                if (!is_vma_resv_set(iter_vma, HPAGE_RESV_OWNER))
-                       unmap_hugepage_range(iter_vma,
+                       __unmap_hugepage_range(iter_vma,
                                address, address + huge_page_size(h),
                                page);
        }
+       spin_unlock(&mapping->i_mmap_lock);
 
        return 1;
 }
@@ -1998,6 +2302,9 @@ retry_avoidcopy:
                outside_reserve = 1;
 
        page_cache_get(old_page);
+
+       /* Drop page_table_lock as buddy allocator may be called */
+       spin_unlock(&mm->page_table_lock);
        new_page = alloc_huge_page(vma, address, outside_reserve);
 
        if (IS_ERR(new_page)) {
@@ -2015,19 +2322,25 @@ retry_avoidcopy:
                        if (unmap_ref_private(mm, vma, old_page, address)) {
                                BUG_ON(page_count(old_page) != 1);
                                BUG_ON(huge_pte_none(pte));
+                               spin_lock(&mm->page_table_lock);
                                goto retry_avoidcopy;
                        }
                        WARN_ON_ONCE(1);
                }
 
+               /* Caller expects lock to be held */
+               spin_lock(&mm->page_table_lock);
                return -PTR_ERR(new_page);
        }
 
-       spin_unlock(&mm->page_table_lock);
        copy_huge_page(new_page, old_page, address, vma);
        __SetPageUptodate(new_page);
-       spin_lock(&mm->page_table_lock);
 
+       /*
+        * Retake the page_table_lock to check for racing updates
+        * before the page tables are altered
+        */
+       spin_lock(&mm->page_table_lock);
        ptep = huge_pte_offset(mm, address & huge_page_mask(h));
        if (likely(pte_same(huge_ptep_get(ptep), pte))) {
                /* Break COW */
@@ -2245,7 +2558,7 @@ int hugetlb_fault(struct mm_struct *mm, struct vm_area_struct *vma,
        entry = pte_mkyoung(entry);
        if (huge_ptep_set_access_flags(vma, address, ptep, entry,
                                                flags & FAULT_FLAG_WRITE))
-               update_mmu_cache(vma, address, entry);
+               update_mmu_cache(vma, address, ptep);
 
 out_page_table_lock:
        spin_unlock(&mm->page_table_lock);