bootmem: fix aligning of node-relative indexes and offsets
[linux-2.6.git] / mm / bootmem.c
index e023c68b025555b6110e8d6352f9ad19e5c7f7f7..ad8eec6e44a85a9593c781ce252b127cb96992cd 100644 (file)
@@ -405,6 +405,29 @@ int __init reserve_bootmem(unsigned long addr, unsigned long size,
 }
 #endif /* !CONFIG_HAVE_ARCH_BOOTMEM_NODE */
 
+static unsigned long align_idx(struct bootmem_data *bdata, unsigned long idx,
+                       unsigned long step)
+{
+       unsigned long base = bdata->node_min_pfn;
+
+       /*
+        * Align the index with respect to the node start so that the
+        * combination of both satisfies the requested alignment.
+        */
+
+       return ALIGN(base + idx, step) - base;
+}
+
+static unsigned long align_off(struct bootmem_data *bdata, unsigned long off,
+                       unsigned long align)
+{
+       unsigned long base = PFN_PHYS(bdata->node_min_pfn);
+
+       /* Same as align_idx for byte offsets */
+
+       return ALIGN(base + off, align) - base;
+}
+
 static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
                                unsigned long size, unsigned long align,
                                unsigned long goal, unsigned long limit)
@@ -441,7 +464,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
        else
                start = ALIGN(min, step);
 
-       sidx = start - bdata->node_min_pfn;;
+       sidx = start - bdata->node_min_pfn;
        midx = max - bdata->node_min_pfn;
 
        if (bdata->hint_idx > sidx) {
@@ -450,7 +473,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
                 * catch the fallback below.
                 */
                fallback = sidx + 1;
-               sidx = ALIGN(bdata->hint_idx, step);
+               sidx = align_idx(bdata, bdata->hint_idx, step);
        }
 
        while (1) {
@@ -459,7 +482,7 @@ static void * __init alloc_bootmem_core(struct bootmem_data *bdata,
                unsigned long eidx, i, start_off, end_off;
 find_block:
                sidx = find_next_zero_bit(bdata->node_bootmem_map, midx, sidx);
-               sidx = ALIGN(sidx, step);
+               sidx = align_idx(bdata, sidx, step);
                eidx = sidx + PFN_UP(size);
 
                if (sidx >= midx || eidx > midx)
@@ -467,7 +490,7 @@ find_block:
 
                for (i = sidx; i < eidx; i++)
                        if (test_bit(i, bdata->node_bootmem_map)) {
-                               sidx = ALIGN(i, step);
+                               sidx = align_idx(bdata, i, step);
                                if (sidx == i)
                                        sidx += step;
                                goto find_block;
@@ -475,7 +498,7 @@ find_block:
 
                if (bdata->last_end_off & (PAGE_SIZE - 1) &&
                                PFN_DOWN(bdata->last_end_off) + 1 == sidx)
-                       start_off = ALIGN(bdata->last_end_off, align);
+                       start_off = align_off(bdata, bdata->last_end_off, align);
                else
                        start_off = PFN_PHYS(sidx);
 
@@ -499,7 +522,7 @@ find_block:
        }
 
        if (fallback) {
-               sidx = ALIGN(fallback - 1, step);
+               sidx = align_idx(bdata, fallback - 1, step);
                fallback = 0;
                goto find_block;
        }