media: video: tegra: sh532u: fix out-of-bounds read
[linux-2.6.git] / mm / bootmem.c
index e023c68..01d5a4b 100644 (file)
  */
 #include <linux/init.h>
 #include <linux/pfn.h>
+#include <linux/slab.h>
 #include <linux/bootmem.h>
 #include <linux/module.h>
+#include <linux/kmemleak.h>
+#include <linux/range.h>
+#include <linux/memblock.h>
 
 #include <asm/bug.h>
 #include <asm/io.h>
 
 #include "internal.h"
 
+#ifndef CONFIG_NEED_MULTIPLE_NODES
+struct pglist_data __refdata contig_page_data = {
+       .bdata = &bootmem_node_data[0]
+};
+EXPORT_SYMBOL(contig_page_data);
+#endif
+
 unsigned long max_low_pfn;
 unsigned long min_low_pfn;
 unsigned long max_pfn;
 
-#ifdef CONFIG_CRASH_DUMP
-/*
- * If we have booted due to a crash, max_pfn will be a very low value. We need
- * to know the amount of memory that the previous kernel used.
- */
-unsigned long saved_max_pfn;
-#endif
-
 bootmem_data_t bootmem_node_data[MAX_NUMNODES] __initdata;
 
 static struct list_head bdata_list __initdata = LIST_HEAD_INIT(bdata_list);
@@ -48,7 +51,7 @@ early_param("bootmem_debug", bootmem_debug_setup);
        if (unlikely(bootmem_debug))                    \
                printk(KERN_INFO                        \
                        "bootmem::%s " fmt,             \
-                       __FUNCTION__, ## args);         \
+                       __func__, ## args);             \
 })
 
 static unsigned long __init bootmap_bytes(unsigned long pages)
@@ -142,6 +145,30 @@ unsigned long __init init_bootmem(unsigned long start, unsigned long pages)
        return init_bootmem_core(NODE_DATA(0)->bdata, start, 0, pages);
 }
 
+/*
+ * free_bootmem_late - free bootmem pages directly to page allocator
+ * @addr: starting address of the range
+ * @size: size of the range in bytes
+ *
+ * This is only useful when the bootmem allocator has already been torn
+ * down, but we are still initializing the system.  Pages are given directly
+ * to the page allocator, no bootmem metadata is updated because it is gone.
+ */
+void __init free_bootmem_late(unsigned long addr, unsigned long size)
+{
+       unsigned long cursor, end;
+
+       kmemleak_free_part(__va(addr), size);
+
+       cursor = PFN_UP(addr);
+       end = PFN_DOWN(addr + size);
+
+       for (; cursor < end; cursor++) {
+               __free_pages_bootmem(pfn_to_page(cursor), 0);
+               totalram_pages++;
+       }
+}
+
 static unsigned long __init free_all_bootmem_core(bootmem_data_t *bdata)
 {
        int aligned;
@@ -222,7 +249,13 @@ unsigned long __init free_all_bootmem_node(pg_data_t *pgdat)
  */
 unsigned long __init free_all_bootmem(void)
 {
-       return free_all_bootmem_core(NODE_DATA(0)->bdata);
+       unsigned long total_pages = 0;
+       bootmem_data_t *bdata;
+
+       list_for_each_entry(bdata, &bdata_list, list)
+               total_pages += free_all_bootmem_core(bdata);
+
+       return total_pages;
 }
 
 static void __init __free(bootmem_data_t *bdata,
@@ -335,6 +368,8 @@ void __init free_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
 {
        unsigned long start, end;
 
+       kmemleak_free_part(__va(physaddr), size);
+
        start = PFN_UP(physaddr);
        end = PFN_DOWN(physaddr + size);
 
@@ -354,6 +389,8 @@ void __init free_bootmem(unsigned long addr, unsigned long size)
 {
        unsigned long start, end;
 
+       kmemleak_free_part(__va(addr), size);
+
        start = PFN_UP(addr);
        end = PFN_DOWN(addr + size);
 
@@ -382,7 +419,6 @@ int __init reserve_bootmem_node(pg_data_t *pgdat, unsigned long physaddr,
        return mark_bootmem_node(pgdat->bdata, start, end, 1, flags);
 }
 
-#ifndef CONFIG_HAVE_ARCH_BOOTMEM_NODE
 /**
  * reserve_bootmem - mark a page range as usable
  * @addr: starting address of the range
@@ -403,15 +439,47 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size,
 
        return mark_bootmem(start, end, 1, flags);
 }
-#endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
+
+int __weak __init reserve_bootmem_generic(unsigned long phys, unsigned long len,
+                                  int flags)
+{
+       return reserve_bootmem(phys, len, flags);
+}
+
+static unsigned long __init align_idx(struct bootmem_data *bdata,
+                                     unsigned long idx, unsigned long step)
+{
+       unsigned long base = bdata->node_min_pfn;
+
+       /*
+        * Align the index with respect to the node start so that the
+        * combination of both satisfies the requested alignment.
+        */
+
+       return ALIGN(base + idx, step) - base;
+}
+
+static unsigned long __init align_off(struct bootmem_data *bdata,
+                                     unsigned long off, unsigned long align)
+{
+       unsigned long base = PFN_PHYS(bdata->node_min_pfn);
+
+       /* Same as align_idx for byte offsets */
+
+       return ALIGN(base + off, align) - base;
+}
 
 static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
-                               unsigned long size, unsigned long align,
-                               unsigned long goal, unsigned long limit)
+                                       unsigned long size, unsigned long align,
+                                       unsigned long goal, unsigned long limit)
 {
        unsigned long fallback = 0;
        unsigned long min, max, start, sidx, midx, step;
 
+       bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
+               bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
+               align, goal, limit);
+
        BUG_ON(!size);
        BUG_ON(align & (align - 1));
        BUG_ON(limit && goal + size > limit);
@@ -419,10 +487,6 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
        if (!bdata->node_bootmem_map)
                return NULL;
 
-       bdebug("nid=%td size=%lx [%lu pages] align=%lx goal=%lx limit=%lx\n",
-               bdata - bootmem_node_data, size, PAGE_ALIGN(size) >> PAGE_SHIFT,
-               align, goal, limit);
-
        min = bdata->node_min_pfn;
        max = bdata->node_low_pfn;
 
@@ -441,7 +505,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
        else
                start = ALIGN(min, step);
 
-       sidx = start - bdata->node_min_pfn;;
+       sidx = start - bdata->node_min_pfn;
        midx = max - bdata->node_min_pfn;
 
        if (bdata->hint_idx > sidx) {
@@ -450,7 +514,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
                 * catch the fallback below.
                 */
                fallback = sidx + 1;
-               sidx = ALIGN(bdata->hint_idx, step);
+               sidx = align_idx(bdata, bdata->hint_idx, step);
        }
 
        while (1) {
@@ -459,7 +523,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
                unsigned long eidx, i, start_off, end_off;
 find_block:
                sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
-               sidx = ALIGN(sidx, step);
+               sidx = align_idx(bdata, sidx, step);
                eidx = sidx + PFN_UP(size);
 
                if (sidx >= midx || eidx > midx)
@@ -467,7 +531,7 @@ find_block:
 
                for (i = sidx; i < eidx; i++)
                        if (test_bit(i, bdata->node_bootmem_map)) {
-                               sidx = ALIGN(i, step);
+                               sidx = align_idx(bdata, i, step);
                                if (sidx == i)
                                        sidx += step;
                                goto find_block;
@@ -475,7 +539,7 @@ find_block:
 
                if (bdata->last_end_off & (PAGE_SIZE - 1) &&
                                PFN_DOWN(bdata->last_end_off) + 1 == sidx)
-                       start_off = ALIGN(bdata->last_end_off, align);
+                       start_off = align_off(bdata, bdata->last_end_off, align);
                else
                        start_off = PFN_PHYS(sidx);
 
@@ -495,11 +559,16 @@ find_block:
                region = phys_to_virt(PFN_PHYS(bdata->node_min_pfn) +
                                start_off);
                memset(region, 0, size);
+               /*
+                * The min_count is set to 0 so that bootmem allocated blocks
+                * are never reported as leaks.
+                */
+               kmemleak_alloc(region, size, 0, 0);
                return region;
        }
 
        if (fallback) {
-               sidx = ALIGN(fallback - 1, step);
+               sidx = align_idx(bdata, fallback - 1, step);
                fallback = 0;
                goto find_block;
        }
@@ -507,17 +576,41 @@ find_block:
        return NULL;
 }
 
+static void * __init alloc_arch_preferred_bootmem(bootmem_data_t *bdata,
+                                       unsigned long size, unsigned long align,
+                                       unsigned long goal, unsigned long limit)
+{
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc(size, GFP_NOWAIT);
+
+#ifdef CONFIG_HAVE_ARCH_BOOTMEM
+       {
+               bootmem_data_t *p_bdata;
+
+               p_bdata = bootmem_arch_preferred_node(bdata, size, align,
+                                                       goal, limit);
+               if (p_bdata)
+                       return alloc_bootmem_core(p_bdata, size, align,
+                                                       goal, limit);
+       }
+#endif
+       return NULL;
+}
+
 static void * __init ___alloc_bootmem_nopanic(unsigned long size,
                                        unsigned long align,
                                        unsigned long goal,
                                        unsigned long limit)
 {
        bootmem_data_t *bdata;
+       void *region;
 
 restart:
-       list_for_each_entry(bdata, &bdata_list, list) {
-               void *region;
+       region = alloc_arch_preferred_bootmem(NULL, size, align, goal, limit);
+       if (region)
+               return region;
 
+       list_for_each_entry(bdata, &bdata_list, list) {
                if (goal && bdata->node_low_pfn <= PFN_DOWN(goal))
                        continue;
                if (limit && bdata->node_min_pfn >= PFN_DOWN(limit))
@@ -552,7 +645,9 @@ restart:
 void * __init __alloc_bootmem_nopanic(unsigned long size, unsigned long align,
                                        unsigned long goal)
 {
-       return ___alloc_bootmem_nopanic(size, align, goal, 0);
+       unsigned long limit = 0;
+
+       return ___alloc_bootmem_nopanic(size, align, goal, limit);
 }
 
 static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
@@ -586,7 +681,9 @@ static void * __init ___alloc_bootmem(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem(unsigned long size, unsigned long align,
                              unsigned long goal)
 {
-       return ___alloc_bootmem(size, align, goal, 0);
+       unsigned long limit = 0;
+
+       return ___alloc_bootmem(size, align, goal, limit);
 }
 
 static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
@@ -595,6 +692,10 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 {
        void *ptr;
 
+       ptr = alloc_arch_preferred_bootmem(bdata, size, align, goal, limit);
+       if (ptr)
+               return ptr;
+
        ptr = alloc_bootmem_core(bdata, size, align, goal, limit);
        if (ptr)
                return ptr;
@@ -620,7 +721,39 @@ static void * __init ___alloc_bootmem_node(bootmem_data_t *bdata,
 void * __init __alloc_bootmem_node(pg_data_t *pgdat, unsigned long size,
                                   unsigned long align, unsigned long goal)
 {
-       return ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+       return  ___alloc_bootmem_node(pgdat->bdata, size, align, goal, 0);
+}
+
+void * __init __alloc_bootmem_node_high(pg_data_t *pgdat, unsigned long size,
+                                  unsigned long align, unsigned long goal)
+{
+#ifdef MAX_DMA32_PFN
+       unsigned long end_pfn;
+
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+       /* update goal according ...MAX_DMA32_PFN */
+       end_pfn = pgdat->node_start_pfn + pgdat->node_spanned_pages;
+
+       if (end_pfn > MAX_DMA32_PFN + (128 >> (20 - PAGE_SHIFT)) &&
+           (goal >> PAGE_SHIFT) < MAX_DMA32_PFN) {
+               void *ptr;
+               unsigned long new_goal;
+
+               new_goal = MAX_DMA32_PFN << PAGE_SHIFT;
+               ptr = alloc_bootmem_core(pgdat->bdata, size, align,
+                                                new_goal, 0);
+               if (ptr)
+                       return ptr;
+       }
+#endif
+
+       return __alloc_bootmem_node(pgdat, size, align, goal);
+
 }
 
 #ifdef CONFIG_SPARSEMEM
@@ -651,6 +784,13 @@ void * __init __alloc_bootmem_node_nopanic(pg_data_t *pgdat, unsigned long size,
 {
        void *ptr;
 
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
+       ptr = alloc_arch_preferred_bootmem(pgdat->bdata, size, align, goal, 0);
+       if (ptr)
+               return ptr;
+
        ptr = alloc_bootmem_core(pgdat->bdata, size, align, goal, 0);
        if (ptr)
                return ptr;
@@ -699,6 +839,9 @@ void * __init __alloc_bootmem_low(unsigned long size, unsigned long align,
 void * __init __alloc_bootmem_low_node(pg_data_t *pgdat, unsigned long size,
                                       unsigned long align, unsigned long goal)
 {
+       if (WARN_ON_ONCE(slab_is_available()))
+               return kzalloc_node(size, GFP_NOWAIT, pgdat->node_id);
+
        return ___alloc_bootmem_node(pgdat->bdata, size, align,
                                goal, ARCH_LOW_ADDRESS_LIMIT);
 }