sparsemem: Put usemap for one node together
[linux-2.6.git] / mm / sparse.c
1 /*
2  * sparse memory mappings.
3  */
4 #include <linux/mm.h>
5 #include <linux/mmzone.h>
6 #include <linux/bootmem.h>
7 #include <linux/highmem.h>
8 #include <linux/module.h>
9 #include <linux/spinlock.h>
10 #include <linux/vmalloc.h>
11 #include "internal.h"
12 #include <asm/dma.h>
13 #include <asm/pgalloc.h>
14 #include <asm/pgtable.h>
15
16 /*
17  * Permanent SPARSEMEM data:
18  *
19  * 1) mem_section       - memory sections, mem_map's for valid memory
20  */
21 #ifdef CONFIG_SPARSEMEM_EXTREME
22 struct mem_section *mem_section[NR_SECTION_ROOTS]
23         ____cacheline_internodealigned_in_smp;
24 #else
25 struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT]
26         ____cacheline_internodealigned_in_smp;
27 #endif
28 EXPORT_SYMBOL(mem_section);
29
30 #ifdef NODE_NOT_IN_PAGE_FLAGS
31 /*
32  * If we did not store the node number in the page then we have to
33  * do a lookup in the section_to_node_table in order to find which
34  * node the page belongs to.
35  */
36 #if MAX_NUMNODES <= 256
37 static u8 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
38 #else
39 static u16 section_to_node_table[NR_MEM_SECTIONS] __cacheline_aligned;
40 #endif
41
42 int page_to_nid(struct page *page)
43 {
44         return section_to_node_table[page_to_section(page)];
45 }
46 EXPORT_SYMBOL(page_to_nid);
47
48 static void set_section_nid(unsigned long section_nr, int nid)
49 {
50         section_to_node_table[section_nr] = nid;
51 }
52 #else /* !NODE_NOT_IN_PAGE_FLAGS */
53 static inline void set_section_nid(unsigned long section_nr, int nid)
54 {
55 }
56 #endif
57
58 #ifdef CONFIG_SPARSEMEM_EXTREME
59 static struct mem_section noinline __init_refok *sparse_index_alloc(int nid)
60 {
61         struct mem_section *section = NULL;
62         unsigned long array_size = SECTIONS_PER_ROOT *
63                                    sizeof(struct mem_section);
64
65         if (slab_is_available()) {
66                 if (node_state(nid, N_HIGH_MEMORY))
67                         section = kmalloc_node(array_size, GFP_KERNEL, nid);
68                 else
69                         section = kmalloc(array_size, GFP_KERNEL);
70         } else
71                 section = alloc_bootmem_node(NODE_DATA(nid), array_size);
72
73         if (section)
74                 memset(section, 0, array_size);
75
76         return section;
77 }
78
79 static int __meminit sparse_index_init(unsigned long section_nr, int nid)
80 {
81         static DEFINE_SPINLOCK(index_init_lock);
82         unsigned long root = SECTION_NR_TO_ROOT(section_nr);
83         struct mem_section *section;
84         int ret = 0;
85
86         if (mem_section[root])
87                 return -EEXIST;
88
89         section = sparse_index_alloc(nid);
90         if (!section)
91                 return -ENOMEM;
92         /*
93          * This lock keeps two different sections from
94          * reallocating for the same index
95          */
96         spin_lock(&index_init_lock);
97
98         if (mem_section[root]) {
99                 ret = -EEXIST;
100                 goto out;
101         }
102
103         mem_section[root] = section;
104 out:
105         spin_unlock(&index_init_lock);
106         return ret;
107 }
108 #else /* !SPARSEMEM_EXTREME */
109 static inline int sparse_index_init(unsigned long section_nr, int nid)
110 {
111         return 0;
112 }
113 #endif
114
115 /*
116  * Although written for the SPARSEMEM_EXTREME case, this happens
117  * to also work for the flat array case because
118  * NR_SECTION_ROOTS==NR_MEM_SECTIONS.
119  */
120 int __section_nr(struct mem_section* ms)
121 {
122         unsigned long root_nr;
123         struct mem_section* root;
124
125         for (root_nr = 0; root_nr < NR_SECTION_ROOTS; root_nr++) {
126                 root = __nr_to_section(root_nr * SECTIONS_PER_ROOT);
127                 if (!root)
128                         continue;
129
130                 if ((ms >= root) && (ms < (root + SECTIONS_PER_ROOT)))
131                      break;
132         }
133
134         return (root_nr * SECTIONS_PER_ROOT) + (ms - root);
135 }
136
137 /*
138  * During early boot, before section_mem_map is used for an actual
139  * mem_map, we use section_mem_map to store the section's NUMA
140  * node.  This keeps us from having to use another data structure.  The
141  * node information is cleared just before we store the real mem_map.
142  */
143 static inline unsigned long sparse_encode_early_nid(int nid)
144 {
145         return (nid << SECTION_NID_SHIFT);
146 }
147
148 static inline int sparse_early_nid(struct mem_section *section)
149 {
150         return (section->section_mem_map >> SECTION_NID_SHIFT);
151 }
152
153 /* Validate the physical addressing limitations of the model */
154 void __meminit mminit_validate_memmodel_limits(unsigned long *start_pfn,
155                                                 unsigned long *end_pfn)
156 {
157         unsigned long max_sparsemem_pfn = 1UL << (MAX_PHYSMEM_BITS-PAGE_SHIFT);
158
159         /*
160          * Sanity checks - do not allow an architecture to pass
161          * in larger pfns than the maximum scope of sparsemem:
162          */
163         if (*start_pfn > max_sparsemem_pfn) {
164                 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
165                         "Start of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
166                         *start_pfn, *end_pfn, max_sparsemem_pfn);
167                 WARN_ON_ONCE(1);
168                 *start_pfn = max_sparsemem_pfn;
169                 *end_pfn = max_sparsemem_pfn;
170         } else if (*end_pfn > max_sparsemem_pfn) {
171                 mminit_dprintk(MMINIT_WARNING, "pfnvalidation",
172                         "End of range %lu -> %lu exceeds SPARSEMEM max %lu\n",
173                         *start_pfn, *end_pfn, max_sparsemem_pfn);
174                 WARN_ON_ONCE(1);
175                 *end_pfn = max_sparsemem_pfn;
176         }
177 }
178
179 /* Record a memory area against a node. */
180 void __init memory_present(int nid, unsigned long start, unsigned long end)
181 {
182         unsigned long pfn;
183
184         start &= PAGE_SECTION_MASK;
185         mminit_validate_memmodel_limits(&start, &end);
186         for (pfn = start; pfn < end; pfn += PAGES_PER_SECTION) {
187                 unsigned long section = pfn_to_section_nr(pfn);
188                 struct mem_section *ms;
189
190                 sparse_index_init(section, nid);
191                 set_section_nid(section, nid);
192
193                 ms = __nr_to_section(section);
194                 if (!ms->section_mem_map)
195                         ms->section_mem_map = sparse_encode_early_nid(nid) |
196                                                         SECTION_MARKED_PRESENT;
197         }
198 }
199
200 /*
201  * Only used by the i386 NUMA architecures, but relatively
202  * generic code.
203  */
204 unsigned long __init node_memmap_size_bytes(int nid, unsigned long start_pfn,
205                                                      unsigned long end_pfn)
206 {
207         unsigned long pfn;
208         unsigned long nr_pages = 0;
209
210         mminit_validate_memmodel_limits(&start_pfn, &end_pfn);
211         for (pfn = start_pfn; pfn < end_pfn; pfn += PAGES_PER_SECTION) {
212                 if (nid != early_pfn_to_nid(pfn))
213                         continue;
214
215                 if (pfn_present(pfn))
216                         nr_pages += PAGES_PER_SECTION;
217         }
218
219         return nr_pages * sizeof(struct page);
220 }
221
222 /*
223  * Subtle, we encode the real pfn into the mem_map such that
224  * the identity pfn - section_mem_map will return the actual
225  * physical page frame number.
226  */
227 static unsigned long sparse_encode_mem_map(struct page *mem_map, unsigned long pnum)
228 {
229         return (unsigned long)(mem_map - (section_nr_to_pfn(pnum)));
230 }
231
232 /*
233  * Decode mem_map from the coded memmap
234  */
235 struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum)
236 {
237         /* mask off the extra low bits of information */
238         coded_mem_map &= SECTION_MAP_MASK;
239         return ((struct page *)coded_mem_map) + section_nr_to_pfn(pnum);
240 }
241
242 static int __meminit sparse_init_one_section(struct mem_section *ms,
243                 unsigned long pnum, struct page *mem_map,
244                 unsigned long *pageblock_bitmap)
245 {
246         if (!present_section(ms))
247                 return -EINVAL;
248
249         ms->section_mem_map &= ~SECTION_MAP_MASK;
250         ms->section_mem_map |= sparse_encode_mem_map(mem_map, pnum) |
251                                                         SECTION_HAS_MEM_MAP;
252         ms->pageblock_flags = pageblock_bitmap;
253
254         return 1;
255 }
256
257 unsigned long usemap_size(void)
258 {
259         unsigned long size_bytes;
260         size_bytes = roundup(SECTION_BLOCKFLAGS_BITS, 8) / 8;
261         size_bytes = roundup(size_bytes, sizeof(unsigned long));
262         return size_bytes;
263 }
264
265 #ifdef CONFIG_MEMORY_HOTPLUG
266 static unsigned long *__kmalloc_section_usemap(void)
267 {
268         return kmalloc(usemap_size(), GFP_KERNEL);
269 }
270 #endif /* CONFIG_MEMORY_HOTPLUG */
271
272 #ifdef CONFIG_MEMORY_HOTREMOVE
273 static unsigned long * __init
274 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
275                                          unsigned long count)
276 {
277         unsigned long section_nr;
278
279         /*
280          * A page may contain usemaps for other sections preventing the
281          * page being freed and making a section unremovable while
282          * other sections referencing the usemap retmain active. Similarly,
283          * a pgdat can prevent a section being removed. If section A
284          * contains a pgdat and section B contains the usemap, both
285          * sections become inter-dependent. This allocates usemaps
286          * from the same section as the pgdat where possible to avoid
287          * this problem.
288          */
289         section_nr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
290         return alloc_bootmem_section(usemap_size() * count, section_nr);
291 }
292
293 static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
294 {
295         unsigned long usemap_snr, pgdat_snr;
296         static unsigned long old_usemap_snr = NR_MEM_SECTIONS;
297         static unsigned long old_pgdat_snr = NR_MEM_SECTIONS;
298         struct pglist_data *pgdat = NODE_DATA(nid);
299         int usemap_nid;
300
301         usemap_snr = pfn_to_section_nr(__pa(usemap) >> PAGE_SHIFT);
302         pgdat_snr = pfn_to_section_nr(__pa(pgdat) >> PAGE_SHIFT);
303         if (usemap_snr == pgdat_snr)
304                 return;
305
306         if (old_usemap_snr == usemap_snr && old_pgdat_snr == pgdat_snr)
307                 /* skip redundant message */
308                 return;
309
310         old_usemap_snr = usemap_snr;
311         old_pgdat_snr = pgdat_snr;
312
313         usemap_nid = sparse_early_nid(__nr_to_section(usemap_snr));
314         if (usemap_nid != nid) {
315                 printk(KERN_INFO
316                        "node %d must be removed before remove section %ld\n",
317                        nid, usemap_snr);
318                 return;
319         }
320         /*
321          * There is a circular dependency.
322          * Some platforms allow un-removable section because they will just
323          * gather other removable sections for dynamic partitioning.
324          * Just notify un-removable section's number here.
325          */
326         printk(KERN_INFO "Section %ld and %ld (node %d)", usemap_snr,
327                pgdat_snr, nid);
328         printk(KERN_CONT
329                " have a circular dependency on usemap and pgdat allocations\n");
330 }
331 #else
332 static unsigned long * __init
333 sparse_early_usemaps_alloc_pgdat_section(struct pglist_data *pgdat,
334                                          unsigned long count)
335 {
336         return NULL;
337 }
338
339 static void __init check_usemap_section_nr(int nid, unsigned long *usemap)
340 {
341 }
342 #endif /* CONFIG_MEMORY_HOTREMOVE */
343
344 static void __init sparse_early_usemaps_alloc_node(unsigned long**usemap_map,
345                                  unsigned long pnum_begin,
346                                  unsigned long pnum_end,
347                                  unsigned long usemap_count, int nodeid)
348 {
349         void *usemap;
350         unsigned long pnum;
351         int size = usemap_size();
352
353         usemap = sparse_early_usemaps_alloc_pgdat_section(NODE_DATA(nodeid),
354                                                                  usemap_count);
355         if (usemap) {
356                 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
357                         if (!present_section_nr(pnum))
358                                 continue;
359                         usemap_map[pnum] = usemap;
360                         usemap += size;
361                 }
362                 return;
363         }
364
365         usemap = alloc_bootmem_node(NODE_DATA(nodeid), size * usemap_count);
366         if (usemap) {
367                 for (pnum = pnum_begin; pnum < pnum_end; pnum++) {
368                         if (!present_section_nr(pnum))
369                                 continue;
370                         usemap_map[pnum] = usemap;
371                         usemap += size;
372                         check_usemap_section_nr(nodeid, usemap_map[pnum]);
373                 }
374                 return;
375         }
376
377         printk(KERN_WARNING "%s: allocation failed\n", __func__);
378 }
379
380 #ifndef CONFIG_SPARSEMEM_VMEMMAP
381 struct page __init *sparse_mem_map_populate(unsigned long pnum, int nid)
382 {
383         struct page *map;
384
385         map = alloc_remap(nid, sizeof(struct page) * PAGES_PER_SECTION);
386         if (map)
387                 return map;
388
389         map = alloc_bootmem_pages_node(NODE_DATA(nid),
390                        PAGE_ALIGN(sizeof(struct page) * PAGES_PER_SECTION));
391         return map;
392 }
393 #endif /* !CONFIG_SPARSEMEM_VMEMMAP */
394
395 static struct page __init *sparse_early_mem_map_alloc(unsigned long pnum)
396 {
397         struct page *map;
398         struct mem_section *ms = __nr_to_section(pnum);
399         int nid = sparse_early_nid(ms);
400
401         map = sparse_mem_map_populate(pnum, nid);
402         if (map)
403                 return map;
404
405         printk(KERN_ERR "%s: sparsemem memory map backing failed "
406                         "some memory will not be available.\n", __func__);
407         ms->section_mem_map = 0;
408         return NULL;
409 }
410
411 void __attribute__((weak)) __meminit vmemmap_populate_print_last(void)
412 {
413 }
414
415 /*
416  * Allocate the accumulated non-linear sections, allocate a mem_map
417  * for each and record the physical to section mapping.
418  */
419 void __init sparse_init(void)
420 {
421         unsigned long pnum;
422         struct page *map;
423         unsigned long *usemap;
424         unsigned long **usemap_map;
425         int size;
426         int nodeid_begin = 0;
427         unsigned long pnum_begin = 0;
428         unsigned long usemap_count;
429
430         /*
431          * map is using big page (aka 2M in x86 64 bit)
432          * usemap is less one page (aka 24 bytes)
433          * so alloc 2M (with 2M align) and 24 bytes in turn will
434          * make next 2M slip to one more 2M later.
435          * then in big system, the memory will have a lot of holes...
436          * here try to allocate 2M pages continously.
437          *
438          * powerpc need to call sparse_init_one_section right after each
439          * sparse_early_mem_map_alloc, so allocate usemap_map at first.
440          */
441         size = sizeof(unsigned long *) * NR_MEM_SECTIONS;
442         usemap_map = alloc_bootmem(size);
443         if (!usemap_map)
444                 panic("can not allocate usemap_map\n");
445
446         for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
447                 struct mem_section *ms;
448
449                 if (!present_section_nr(pnum))
450                         continue;
451                 ms = __nr_to_section(pnum);
452                 nodeid_begin = sparse_early_nid(ms);
453                 pnum_begin = pnum;
454                 break;
455         }
456         usemap_count = 1;
457         for (pnum = pnum_begin + 1; pnum < NR_MEM_SECTIONS; pnum++) {
458                 struct mem_section *ms;
459                 int nodeid;
460
461                 if (!present_section_nr(pnum))
462                         continue;
463                 ms = __nr_to_section(pnum);
464                 nodeid = sparse_early_nid(ms);
465                 if (nodeid == nodeid_begin) {
466                         usemap_count++;
467                         continue;
468                 }
469                 /* ok, we need to take cake of from pnum_begin to pnum - 1*/
470                 sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, pnum,
471                                                  usemap_count, nodeid_begin);
472                 /* new start, update count etc*/
473                 nodeid_begin = nodeid;
474                 pnum_begin = pnum;
475                 usemap_count = 1;
476         }
477         /* ok, last chunk */
478         sparse_early_usemaps_alloc_node(usemap_map, pnum_begin, NR_MEM_SECTIONS,
479                                          usemap_count, nodeid_begin);
480
481         for (pnum = 0; pnum < NR_MEM_SECTIONS; pnum++) {
482                 if (!present_section_nr(pnum))
483                         continue;
484
485                 usemap = usemap_map[pnum];
486                 if (!usemap)
487                         continue;
488
489                 map = sparse_early_mem_map_alloc(pnum);
490                 if (!map)
491                         continue;
492
493                 sparse_init_one_section(__nr_to_section(pnum), pnum, map,
494                                                                 usemap);
495         }
496
497         vmemmap_populate_print_last();
498
499         free_bootmem(__pa(usemap_map), size);
500 }
501
502 #ifdef CONFIG_MEMORY_HOTPLUG
503 #ifdef CONFIG_SPARSEMEM_VMEMMAP
504 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
505                                                  unsigned long nr_pages)
506 {
507         /* This will make the necessary allocations eventually. */
508         return sparse_mem_map_populate(pnum, nid);
509 }
510 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
511 {
512         return; /* XXX: Not implemented yet */
513 }
514 static void free_map_bootmem(struct page *page, unsigned long nr_pages)
515 {
516 }
517 #else
518 static struct page *__kmalloc_section_memmap(unsigned long nr_pages)
519 {
520         struct page *page, *ret;
521         unsigned long memmap_size = sizeof(struct page) * nr_pages;
522
523         page = alloc_pages(GFP_KERNEL|__GFP_NOWARN, get_order(memmap_size));
524         if (page)
525                 goto got_map_page;
526
527         ret = vmalloc(memmap_size);
528         if (ret)
529                 goto got_map_ptr;
530
531         return NULL;
532 got_map_page:
533         ret = (struct page *)pfn_to_kaddr(page_to_pfn(page));
534 got_map_ptr:
535         memset(ret, 0, memmap_size);
536
537         return ret;
538 }
539
540 static inline struct page *kmalloc_section_memmap(unsigned long pnum, int nid,
541                                                   unsigned long nr_pages)
542 {
543         return __kmalloc_section_memmap(nr_pages);
544 }
545
546 static void __kfree_section_memmap(struct page *memmap, unsigned long nr_pages)
547 {
548         if (is_vmalloc_addr(memmap))
549                 vfree(memmap);
550         else
551                 free_pages((unsigned long)memmap,
552                            get_order(sizeof(struct page) * nr_pages));
553 }
554
555 static void free_map_bootmem(struct page *page, unsigned long nr_pages)
556 {
557         unsigned long maps_section_nr, removing_section_nr, i;
558         int magic;
559
560         for (i = 0; i < nr_pages; i++, page++) {
561                 magic = atomic_read(&page->_mapcount);
562
563                 BUG_ON(magic == NODE_INFO);
564
565                 maps_section_nr = pfn_to_section_nr(page_to_pfn(page));
566                 removing_section_nr = page->private;
567
568                 /*
569                  * When this function is called, the removing section is
570                  * logical offlined state. This means all pages are isolated
571                  * from page allocator. If removing section's memmap is placed
572                  * on the same section, it must not be freed.
573                  * If it is freed, page allocator may allocate it which will
574                  * be removed physically soon.
575                  */
576                 if (maps_section_nr != removing_section_nr)
577                         put_page_bootmem(page);
578         }
579 }
580 #endif /* CONFIG_SPARSEMEM_VMEMMAP */
581
582 static void free_section_usemap(struct page *memmap, unsigned long *usemap)
583 {
584         struct page *usemap_page;
585         unsigned long nr_pages;
586
587         if (!usemap)
588                 return;
589
590         usemap_page = virt_to_page(usemap);
591         /*
592          * Check to see if allocation came from hot-plug-add
593          */
594         if (PageSlab(usemap_page)) {
595                 kfree(usemap);
596                 if (memmap)
597                         __kfree_section_memmap(memmap, PAGES_PER_SECTION);
598                 return;
599         }
600
601         /*
602          * The usemap came from bootmem. This is packed with other usemaps
603          * on the section which has pgdat at boot time. Just keep it as is now.
604          */
605
606         if (memmap) {
607                 struct page *memmap_page;
608                 memmap_page = virt_to_page(memmap);
609
610                 nr_pages = PAGE_ALIGN(PAGES_PER_SECTION * sizeof(struct page))
611                         >> PAGE_SHIFT;
612
613                 free_map_bootmem(memmap_page, nr_pages);
614         }
615 }
616
617 /*
618  * returns the number of sections whose mem_maps were properly
619  * set.  If this is <=0, then that means that the passed-in
620  * map was not consumed and must be freed.
621  */
622 int __meminit sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
623                            int nr_pages)
624 {
625         unsigned long section_nr = pfn_to_section_nr(start_pfn);
626         struct pglist_data *pgdat = zone->zone_pgdat;
627         struct mem_section *ms;
628         struct page *memmap;
629         unsigned long *usemap;
630         unsigned long flags;
631         int ret;
632
633         /*
634          * no locking for this, because it does its own
635          * plus, it does a kmalloc
636          */
637         ret = sparse_index_init(section_nr, pgdat->node_id);
638         if (ret < 0 && ret != -EEXIST)
639                 return ret;
640         memmap = kmalloc_section_memmap(section_nr, pgdat->node_id, nr_pages);
641         if (!memmap)
642                 return -ENOMEM;
643         usemap = __kmalloc_section_usemap();
644         if (!usemap) {
645                 __kfree_section_memmap(memmap, nr_pages);
646                 return -ENOMEM;
647         }
648
649         pgdat_resize_lock(pgdat, &flags);
650
651         ms = __pfn_to_section(start_pfn);
652         if (ms->section_mem_map & SECTION_MARKED_PRESENT) {
653                 ret = -EEXIST;
654                 goto out;
655         }
656
657         ms->section_mem_map |= SECTION_MARKED_PRESENT;
658
659         ret = sparse_init_one_section(ms, section_nr, memmap, usemap);
660
661 out:
662         pgdat_resize_unlock(pgdat, &flags);
663         if (ret <= 0) {
664                 kfree(usemap);
665                 __kfree_section_memmap(memmap, nr_pages);
666         }
667         return ret;
668 }
669
670 void sparse_remove_one_section(struct zone *zone, struct mem_section *ms)
671 {
672         struct page *memmap = NULL;
673         unsigned long *usemap = NULL;
674
675         if (ms->section_mem_map) {
676                 usemap = ms->pageblock_flags;
677                 memmap = sparse_decode_mem_map(ms->section_mem_map,
678                                                 __section_nr(ms));
679                 ms->section_mem_map = 0;
680                 ms->pageblock_flags = NULL;
681         }
682
683         free_section_usemap(memmap, usemap);
684 }
685 #endif