Page allocator: clean up pcp draining functions
[linux-2.6.git] / kernel / power / snapshot.c
1 /*
2  * linux/kernel/power/snapshot.c
3  *
4  * This file provides system snapshot/restore functionality for swsusp.
5  *
6  * Copyright (C) 1998-2005 Pavel Machek <pavel@suse.cz>
7  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8  *
9  * This file is released under the GPLv2.
10  *
11  */
12
13 #include <linux/version.h>
14 #include <linux/module.h>
15 #include <linux/mm.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
21 #include <linux/pm.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28
29 #include <asm/uaccess.h>
30 #include <asm/mmu_context.h>
31 #include <asm/pgtable.h>
32 #include <asm/tlbflush.h>
33 #include <asm/io.h>
34
35 #include "power.h"
36
37 static int swsusp_page_is_free(struct page *);
38 static void swsusp_set_page_forbidden(struct page *);
39 static void swsusp_unset_page_forbidden(struct page *);
40
41 /* List of PBEs needed for restoring the pages that were allocated before
42  * the suspend and included in the suspend image, but have also been
43  * allocated by the "resume" kernel, so their contents cannot be written
44  * directly to their "original" page frames.
45  */
46 struct pbe *restore_pblist;
47
48 /* Pointer to an auxiliary buffer (1 page) */
49 static void *buffer;
50
51 /**
52  *      @safe_needed - on resume, for storing the PBE list and the image,
53  *      we can only use memory pages that do not conflict with the pages
54  *      used before suspend.  The unsafe pages have PageNosaveFree set
55  *      and we count them using unsafe_pages.
56  *
57  *      Each allocated image page is marked as PageNosave and PageNosaveFree
58  *      so that swsusp_free() can release it.
59  */
60
61 #define PG_ANY          0
62 #define PG_SAFE         1
63 #define PG_UNSAFE_CLEAR 1
64 #define PG_UNSAFE_KEEP  0
65
66 static unsigned int allocated_unsafe_pages;
67
68 static void *get_image_page(gfp_t gfp_mask, int safe_needed)
69 {
70         void *res;
71
72         res = (void *)get_zeroed_page(gfp_mask);
73         if (safe_needed)
74                 while (res && swsusp_page_is_free(virt_to_page(res))) {
75                         /* The page is unsafe, mark it for swsusp_free() */
76                         swsusp_set_page_forbidden(virt_to_page(res));
77                         allocated_unsafe_pages++;
78                         res = (void *)get_zeroed_page(gfp_mask);
79                 }
80         if (res) {
81                 swsusp_set_page_forbidden(virt_to_page(res));
82                 swsusp_set_page_free(virt_to_page(res));
83         }
84         return res;
85 }
86
87 unsigned long get_safe_page(gfp_t gfp_mask)
88 {
89         return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
90 }
91
92 static struct page *alloc_image_page(gfp_t gfp_mask)
93 {
94         struct page *page;
95
96         page = alloc_page(gfp_mask);
97         if (page) {
98                 swsusp_set_page_forbidden(page);
99                 swsusp_set_page_free(page);
100         }
101         return page;
102 }
103
104 /**
105  *      free_image_page - free page represented by @addr, allocated with
106  *      get_image_page (page flags set by it must be cleared)
107  */
108
109 static inline void free_image_page(void *addr, int clear_nosave_free)
110 {
111         struct page *page;
112
113         BUG_ON(!virt_addr_valid(addr));
114
115         page = virt_to_page(addr);
116
117         swsusp_unset_page_forbidden(page);
118         if (clear_nosave_free)
119                 swsusp_unset_page_free(page);
120
121         __free_page(page);
122 }
123
124 /* struct linked_page is used to build chains of pages */
125
126 #define LINKED_PAGE_DATA_SIZE   (PAGE_SIZE - sizeof(void *))
127
128 struct linked_page {
129         struct linked_page *next;
130         char data[LINKED_PAGE_DATA_SIZE];
131 } __attribute__((packed));
132
133 static inline void
134 free_list_of_pages(struct linked_page *list, int clear_page_nosave)
135 {
136         while (list) {
137                 struct linked_page *lp = list->next;
138
139                 free_image_page(list, clear_page_nosave);
140                 list = lp;
141         }
142 }
143
144 /**
145   *     struct chain_allocator is used for allocating small objects out of
146   *     a linked list of pages called 'the chain'.
147   *
148   *     The chain grows each time when there is no room for a new object in
149   *     the current page.  The allocated objects cannot be freed individually.
150   *     It is only possible to free them all at once, by freeing the entire
151   *     chain.
152   *
153   *     NOTE: The chain allocator may be inefficient if the allocated objects
154   *     are not much smaller than PAGE_SIZE.
155   */
156
157 struct chain_allocator {
158         struct linked_page *chain;      /* the chain */
159         unsigned int used_space;        /* total size of objects allocated out
160                                          * of the current page
161                                          */
162         gfp_t gfp_mask;         /* mask for allocating pages */
163         int safe_needed;        /* if set, only "safe" pages are allocated */
164 };
165
166 static void
167 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
168 {
169         ca->chain = NULL;
170         ca->used_space = LINKED_PAGE_DATA_SIZE;
171         ca->gfp_mask = gfp_mask;
172         ca->safe_needed = safe_needed;
173 }
174
175 static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
176 {
177         void *ret;
178
179         if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
180                 struct linked_page *lp;
181
182                 lp = get_image_page(ca->gfp_mask, ca->safe_needed);
183                 if (!lp)
184                         return NULL;
185
186                 lp->next = ca->chain;
187                 ca->chain = lp;
188                 ca->used_space = 0;
189         }
190         ret = ca->chain->data + ca->used_space;
191         ca->used_space += size;
192         return ret;
193 }
194
195 static void chain_free(struct chain_allocator *ca, int clear_page_nosave)
196 {
197         free_list_of_pages(ca->chain, clear_page_nosave);
198         memset(ca, 0, sizeof(struct chain_allocator));
199 }
200
201 /**
202  *      Data types related to memory bitmaps.
203  *
204  *      Memory bitmap is a structure consiting of many linked lists of
205  *      objects.  The main list's elements are of type struct zone_bitmap
206  *      and each of them corresonds to one zone.  For each zone bitmap
207  *      object there is a list of objects of type struct bm_block that
208  *      represent each blocks of bit chunks in which information is
209  *      stored.
210  *
211  *      struct memory_bitmap contains a pointer to the main list of zone
212  *      bitmap objects, a struct bm_position used for browsing the bitmap,
213  *      and a pointer to the list of pages used for allocating all of the
214  *      zone bitmap objects and bitmap block objects.
215  *
216  *      NOTE: It has to be possible to lay out the bitmap in memory
217  *      using only allocations of order 0.  Additionally, the bitmap is
218  *      designed to work with arbitrary number of zones (this is over the
219  *      top for now, but let's avoid making unnecessary assumptions ;-).
220  *
221  *      struct zone_bitmap contains a pointer to a list of bitmap block
222  *      objects and a pointer to the bitmap block object that has been
223  *      most recently used for setting bits.  Additionally, it contains the
224  *      pfns that correspond to the start and end of the represented zone.
225  *
226  *      struct bm_block contains a pointer to the memory page in which
227  *      information is stored (in the form of a block of bit chunks
228  *      of type unsigned long each).  It also contains the pfns that
229  *      correspond to the start and end of the represented memory area and
230  *      the number of bit chunks in the block.
231  */
232
233 #define BM_END_OF_MAP   (~0UL)
234
235 #define BM_CHUNKS_PER_BLOCK     (PAGE_SIZE / sizeof(long))
236 #define BM_BITS_PER_CHUNK       (sizeof(long) << 3)
237 #define BM_BITS_PER_BLOCK       (PAGE_SIZE << 3)
238
239 struct bm_block {
240         struct bm_block *next;          /* next element of the list */
241         unsigned long start_pfn;        /* pfn represented by the first bit */
242         unsigned long end_pfn;  /* pfn represented by the last bit plus 1 */
243         unsigned int size;      /* number of bit chunks */
244         unsigned long *data;    /* chunks of bits representing pages */
245 };
246
247 struct zone_bitmap {
248         struct zone_bitmap *next;       /* next element of the list */
249         unsigned long start_pfn;        /* minimal pfn in this zone */
250         unsigned long end_pfn;          /* maximal pfn in this zone plus 1 */
251         struct bm_block *bm_blocks;     /* list of bitmap blocks */
252         struct bm_block *cur_block;     /* recently used bitmap block */
253 };
254
255 /* strcut bm_position is used for browsing memory bitmaps */
256
257 struct bm_position {
258         struct zone_bitmap *zone_bm;
259         struct bm_block *block;
260         int chunk;
261         int bit;
262 };
263
264 struct memory_bitmap {
265         struct zone_bitmap *zone_bm_list;       /* list of zone bitmaps */
266         struct linked_page *p_list;     /* list of pages used to store zone
267                                          * bitmap objects and bitmap block
268                                          * objects
269                                          */
270         struct bm_position cur; /* most recently used bit position */
271 };
272
273 /* Functions that operate on memory bitmaps */
274
275 static inline void memory_bm_reset_chunk(struct memory_bitmap *bm)
276 {
277         bm->cur.chunk = 0;
278         bm->cur.bit = -1;
279 }
280
281 static void memory_bm_position_reset(struct memory_bitmap *bm)
282 {
283         struct zone_bitmap *zone_bm;
284
285         zone_bm = bm->zone_bm_list;
286         bm->cur.zone_bm = zone_bm;
287         bm->cur.block = zone_bm->bm_blocks;
288         memory_bm_reset_chunk(bm);
289 }
290
291 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
292
293 /**
294  *      create_bm_block_list - create a list of block bitmap objects
295  */
296
297 static inline struct bm_block *
298 create_bm_block_list(unsigned int nr_blocks, struct chain_allocator *ca)
299 {
300         struct bm_block *bblist = NULL;
301
302         while (nr_blocks-- > 0) {
303                 struct bm_block *bb;
304
305                 bb = chain_alloc(ca, sizeof(struct bm_block));
306                 if (!bb)
307                         return NULL;
308
309                 bb->next = bblist;
310                 bblist = bb;
311         }
312         return bblist;
313 }
314
315 /**
316  *      create_zone_bm_list - create a list of zone bitmap objects
317  */
318
319 static inline struct zone_bitmap *
320 create_zone_bm_list(unsigned int nr_zones, struct chain_allocator *ca)
321 {
322         struct zone_bitmap *zbmlist = NULL;
323
324         while (nr_zones-- > 0) {
325                 struct zone_bitmap *zbm;
326
327                 zbm = chain_alloc(ca, sizeof(struct zone_bitmap));
328                 if (!zbm)
329                         return NULL;
330
331                 zbm->next = zbmlist;
332                 zbmlist = zbm;
333         }
334         return zbmlist;
335 }
336
337 /**
338   *     memory_bm_create - allocate memory for a memory bitmap
339   */
340
341 static int
342 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
343 {
344         struct chain_allocator ca;
345         struct zone *zone;
346         struct zone_bitmap *zone_bm;
347         struct bm_block *bb;
348         unsigned int nr;
349
350         chain_init(&ca, gfp_mask, safe_needed);
351
352         /* Compute the number of zones */
353         nr = 0;
354         for_each_zone(zone)
355                 if (populated_zone(zone))
356                         nr++;
357
358         /* Allocate the list of zones bitmap objects */
359         zone_bm = create_zone_bm_list(nr, &ca);
360         bm->zone_bm_list = zone_bm;
361         if (!zone_bm) {
362                 chain_free(&ca, PG_UNSAFE_CLEAR);
363                 return -ENOMEM;
364         }
365
366         /* Initialize the zone bitmap objects */
367         for_each_zone(zone) {
368                 unsigned long pfn;
369
370                 if (!populated_zone(zone))
371                         continue;
372
373                 zone_bm->start_pfn = zone->zone_start_pfn;
374                 zone_bm->end_pfn = zone->zone_start_pfn + zone->spanned_pages;
375                 /* Allocate the list of bitmap block objects */
376                 nr = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
377                 bb = create_bm_block_list(nr, &ca);
378                 zone_bm->bm_blocks = bb;
379                 zone_bm->cur_block = bb;
380                 if (!bb)
381                         goto Free;
382
383                 nr = zone->spanned_pages;
384                 pfn = zone->zone_start_pfn;
385                 /* Initialize the bitmap block objects */
386                 while (bb) {
387                         unsigned long *ptr;
388
389                         ptr = get_image_page(gfp_mask, safe_needed);
390                         bb->data = ptr;
391                         if (!ptr)
392                                 goto Free;
393
394                         bb->start_pfn = pfn;
395                         if (nr >= BM_BITS_PER_BLOCK) {
396                                 pfn += BM_BITS_PER_BLOCK;
397                                 bb->size = BM_CHUNKS_PER_BLOCK;
398                                 nr -= BM_BITS_PER_BLOCK;
399                         } else {
400                                 /* This is executed only once in the loop */
401                                 pfn += nr;
402                                 bb->size = DIV_ROUND_UP(nr, BM_BITS_PER_CHUNK);
403                         }
404                         bb->end_pfn = pfn;
405                         bb = bb->next;
406                 }
407                 zone_bm = zone_bm->next;
408         }
409         bm->p_list = ca.chain;
410         memory_bm_position_reset(bm);
411         return 0;
412
413  Free:
414         bm->p_list = ca.chain;
415         memory_bm_free(bm, PG_UNSAFE_CLEAR);
416         return -ENOMEM;
417 }
418
419 /**
420   *     memory_bm_free - free memory occupied by the memory bitmap @bm
421   */
422
423 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
424 {
425         struct zone_bitmap *zone_bm;
426
427         /* Free the list of bit blocks for each zone_bitmap object */
428         zone_bm = bm->zone_bm_list;
429         while (zone_bm) {
430                 struct bm_block *bb;
431
432                 bb = zone_bm->bm_blocks;
433                 while (bb) {
434                         if (bb->data)
435                                 free_image_page(bb->data, clear_nosave_free);
436                         bb = bb->next;
437                 }
438                 zone_bm = zone_bm->next;
439         }
440         free_list_of_pages(bm->p_list, clear_nosave_free);
441         bm->zone_bm_list = NULL;
442 }
443
444 /**
445  *      memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
446  *      to given pfn.  The cur_zone_bm member of @bm and the cur_block member
447  *      of @bm->cur_zone_bm are updated.
448  */
449
450 static void memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
451                                 void **addr, unsigned int *bit_nr)
452 {
453         struct zone_bitmap *zone_bm;
454         struct bm_block *bb;
455
456         /* Check if the pfn is from the current zone */
457         zone_bm = bm->cur.zone_bm;
458         if (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
459                 zone_bm = bm->zone_bm_list;
460                 /* We don't assume that the zones are sorted by pfns */
461                 while (pfn < zone_bm->start_pfn || pfn >= zone_bm->end_pfn) {
462                         zone_bm = zone_bm->next;
463
464                         BUG_ON(!zone_bm);
465                 }
466                 bm->cur.zone_bm = zone_bm;
467         }
468         /* Check if the pfn corresponds to the current bitmap block */
469         bb = zone_bm->cur_block;
470         if (pfn < bb->start_pfn)
471                 bb = zone_bm->bm_blocks;
472
473         while (pfn >= bb->end_pfn) {
474                 bb = bb->next;
475
476                 BUG_ON(!bb);
477         }
478         zone_bm->cur_block = bb;
479         pfn -= bb->start_pfn;
480         *bit_nr = pfn % BM_BITS_PER_CHUNK;
481         *addr = bb->data + pfn / BM_BITS_PER_CHUNK;
482 }
483
484 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
485 {
486         void *addr;
487         unsigned int bit;
488
489         memory_bm_find_bit(bm, pfn, &addr, &bit);
490         set_bit(bit, addr);
491 }
492
493 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
494 {
495         void *addr;
496         unsigned int bit;
497
498         memory_bm_find_bit(bm, pfn, &addr, &bit);
499         clear_bit(bit, addr);
500 }
501
502 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
503 {
504         void *addr;
505         unsigned int bit;
506
507         memory_bm_find_bit(bm, pfn, &addr, &bit);
508         return test_bit(bit, addr);
509 }
510
511 /* Two auxiliary functions for memory_bm_next_pfn */
512
513 /* Find the first set bit in the given chunk, if there is one */
514
515 static inline int next_bit_in_chunk(int bit, unsigned long *chunk_p)
516 {
517         bit++;
518         while (bit < BM_BITS_PER_CHUNK) {
519                 if (test_bit(bit, chunk_p))
520                         return bit;
521
522                 bit++;
523         }
524         return -1;
525 }
526
527 /* Find a chunk containing some bits set in given block of bits */
528
529 static inline int next_chunk_in_block(int n, struct bm_block *bb)
530 {
531         n++;
532         while (n < bb->size) {
533                 if (bb->data[n])
534                         return n;
535
536                 n++;
537         }
538         return -1;
539 }
540
541 /**
542  *      memory_bm_next_pfn - find the pfn that corresponds to the next set bit
543  *      in the bitmap @bm.  If the pfn cannot be found, BM_END_OF_MAP is
544  *      returned.
545  *
546  *      It is required to run memory_bm_position_reset() before the first call to
547  *      this function.
548  */
549
550 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
551 {
552         struct zone_bitmap *zone_bm;
553         struct bm_block *bb;
554         int chunk;
555         int bit;
556
557         do {
558                 bb = bm->cur.block;
559                 do {
560                         chunk = bm->cur.chunk;
561                         bit = bm->cur.bit;
562                         do {
563                                 bit = next_bit_in_chunk(bit, bb->data + chunk);
564                                 if (bit >= 0)
565                                         goto Return_pfn;
566
567                                 chunk = next_chunk_in_block(chunk, bb);
568                                 bit = -1;
569                         } while (chunk >= 0);
570                         bb = bb->next;
571                         bm->cur.block = bb;
572                         memory_bm_reset_chunk(bm);
573                 } while (bb);
574                 zone_bm = bm->cur.zone_bm->next;
575                 if (zone_bm) {
576                         bm->cur.zone_bm = zone_bm;
577                         bm->cur.block = zone_bm->bm_blocks;
578                         memory_bm_reset_chunk(bm);
579                 }
580         } while (zone_bm);
581         memory_bm_position_reset(bm);
582         return BM_END_OF_MAP;
583
584  Return_pfn:
585         bm->cur.chunk = chunk;
586         bm->cur.bit = bit;
587         return bb->start_pfn + chunk * BM_BITS_PER_CHUNK + bit;
588 }
589
590 /**
591  *      This structure represents a range of page frames the contents of which
592  *      should not be saved during the suspend.
593  */
594
595 struct nosave_region {
596         struct list_head list;
597         unsigned long start_pfn;
598         unsigned long end_pfn;
599 };
600
601 static LIST_HEAD(nosave_regions);
602
603 /**
604  *      register_nosave_region - register a range of page frames the contents
605  *      of which should not be saved during the suspend (to be used in the early
606  *      initialization code)
607  */
608
609 void __init
610 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
611                          int use_kmalloc)
612 {
613         struct nosave_region *region;
614
615         if (start_pfn >= end_pfn)
616                 return;
617
618         if (!list_empty(&nosave_regions)) {
619                 /* Try to extend the previous region (they should be sorted) */
620                 region = list_entry(nosave_regions.prev,
621                                         struct nosave_region, list);
622                 if (region->end_pfn == start_pfn) {
623                         region->end_pfn = end_pfn;
624                         goto Report;
625                 }
626         }
627         if (use_kmalloc) {
628                 /* during init, this shouldn't fail */
629                 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
630                 BUG_ON(!region);
631         } else
632                 /* This allocation cannot fail */
633                 region = alloc_bootmem_low(sizeof(struct nosave_region));
634         region->start_pfn = start_pfn;
635         region->end_pfn = end_pfn;
636         list_add_tail(&region->list, &nosave_regions);
637  Report:
638         printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
639                 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
640 }
641
642 /*
643  * Set bits in this map correspond to the page frames the contents of which
644  * should not be saved during the suspend.
645  */
646 static struct memory_bitmap *forbidden_pages_map;
647
648 /* Set bits in this map correspond to free page frames. */
649 static struct memory_bitmap *free_pages_map;
650
651 /*
652  * Each page frame allocated for creating the image is marked by setting the
653  * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
654  */
655
656 void swsusp_set_page_free(struct page *page)
657 {
658         if (free_pages_map)
659                 memory_bm_set_bit(free_pages_map, page_to_pfn(page));
660 }
661
662 static int swsusp_page_is_free(struct page *page)
663 {
664         return free_pages_map ?
665                 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
666 }
667
668 void swsusp_unset_page_free(struct page *page)
669 {
670         if (free_pages_map)
671                 memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
672 }
673
674 static void swsusp_set_page_forbidden(struct page *page)
675 {
676         if (forbidden_pages_map)
677                 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
678 }
679
680 int swsusp_page_is_forbidden(struct page *page)
681 {
682         return forbidden_pages_map ?
683                 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
684 }
685
686 static void swsusp_unset_page_forbidden(struct page *page)
687 {
688         if (forbidden_pages_map)
689                 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
690 }
691
692 /**
693  *      mark_nosave_pages - set bits corresponding to the page frames the
694  *      contents of which should not be saved in a given bitmap.
695  */
696
697 static void mark_nosave_pages(struct memory_bitmap *bm)
698 {
699         struct nosave_region *region;
700
701         if (list_empty(&nosave_regions))
702                 return;
703
704         list_for_each_entry(region, &nosave_regions, list) {
705                 unsigned long pfn;
706
707                 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
708                                 region->start_pfn << PAGE_SHIFT,
709                                 region->end_pfn << PAGE_SHIFT);
710
711                 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
712                         if (pfn_valid(pfn))
713                                 memory_bm_set_bit(bm, pfn);
714         }
715 }
716
717 /**
718  *      create_basic_memory_bitmaps - create bitmaps needed for marking page
719  *      frames that should not be saved and free page frames.  The pointers
720  *      forbidden_pages_map and free_pages_map are only modified if everything
721  *      goes well, because we don't want the bits to be used before both bitmaps
722  *      are set up.
723  */
724
725 int create_basic_memory_bitmaps(void)
726 {
727         struct memory_bitmap *bm1, *bm2;
728         int error = 0;
729
730         BUG_ON(forbidden_pages_map || free_pages_map);
731
732         bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
733         if (!bm1)
734                 return -ENOMEM;
735
736         error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
737         if (error)
738                 goto Free_first_object;
739
740         bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
741         if (!bm2)
742                 goto Free_first_bitmap;
743
744         error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
745         if (error)
746                 goto Free_second_object;
747
748         forbidden_pages_map = bm1;
749         free_pages_map = bm2;
750         mark_nosave_pages(forbidden_pages_map);
751
752         pr_debug("PM: Basic memory bitmaps created\n");
753
754         return 0;
755
756  Free_second_object:
757         kfree(bm2);
758  Free_first_bitmap:
759         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
760  Free_first_object:
761         kfree(bm1);
762         return -ENOMEM;
763 }
764
765 /**
766  *      free_basic_memory_bitmaps - free memory bitmaps allocated by
767  *      create_basic_memory_bitmaps().  The auxiliary pointers are necessary
768  *      so that the bitmaps themselves are not referred to while they are being
769  *      freed.
770  */
771
772 void free_basic_memory_bitmaps(void)
773 {
774         struct memory_bitmap *bm1, *bm2;
775
776         BUG_ON(!(forbidden_pages_map && free_pages_map));
777
778         bm1 = forbidden_pages_map;
779         bm2 = free_pages_map;
780         forbidden_pages_map = NULL;
781         free_pages_map = NULL;
782         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
783         kfree(bm1);
784         memory_bm_free(bm2, PG_UNSAFE_CLEAR);
785         kfree(bm2);
786
787         pr_debug("PM: Basic memory bitmaps freed\n");
788 }
789
790 /**
791  *      snapshot_additional_pages - estimate the number of additional pages
792  *      be needed for setting up the suspend image data structures for given
793  *      zone (usually the returned value is greater than the exact number)
794  */
795
796 unsigned int snapshot_additional_pages(struct zone *zone)
797 {
798         unsigned int res;
799
800         res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
801         res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
802         return 2 * res;
803 }
804
805 #ifdef CONFIG_HIGHMEM
806 /**
807  *      count_free_highmem_pages - compute the total number of free highmem
808  *      pages, system-wide.
809  */
810
811 static unsigned int count_free_highmem_pages(void)
812 {
813         struct zone *zone;
814         unsigned int cnt = 0;
815
816         for_each_zone(zone)
817                 if (populated_zone(zone) && is_highmem(zone))
818                         cnt += zone_page_state(zone, NR_FREE_PAGES);
819
820         return cnt;
821 }
822
823 /**
824  *      saveable_highmem_page - Determine whether a highmem page should be
825  *      included in the suspend image.
826  *
827  *      We should save the page if it isn't Nosave or NosaveFree, or Reserved,
828  *      and it isn't a part of a free chunk of pages.
829  */
830
831 static struct page *saveable_highmem_page(unsigned long pfn)
832 {
833         struct page *page;
834
835         if (!pfn_valid(pfn))
836                 return NULL;
837
838         page = pfn_to_page(pfn);
839
840         BUG_ON(!PageHighMem(page));
841
842         if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page) ||
843             PageReserved(page))
844                 return NULL;
845
846         return page;
847 }
848
849 /**
850  *      count_highmem_pages - compute the total number of saveable highmem
851  *      pages.
852  */
853
854 unsigned int count_highmem_pages(void)
855 {
856         struct zone *zone;
857         unsigned int n = 0;
858
859         for_each_zone(zone) {
860                 unsigned long pfn, max_zone_pfn;
861
862                 if (!is_highmem(zone))
863                         continue;
864
865                 mark_free_pages(zone);
866                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
867                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
868                         if (saveable_highmem_page(pfn))
869                                 n++;
870         }
871         return n;
872 }
873 #else
874 static inline void *saveable_highmem_page(unsigned long pfn) { return NULL; }
875 #endif /* CONFIG_HIGHMEM */
876
877 /**
878  *      saveable - Determine whether a non-highmem page should be included in
879  *      the suspend image.
880  *
881  *      We should save the page if it isn't Nosave, and is not in the range
882  *      of pages statically defined as 'unsaveable', and it isn't a part of
883  *      a free chunk of pages.
884  */
885
886 static struct page *saveable_page(unsigned long pfn)
887 {
888         struct page *page;
889
890         if (!pfn_valid(pfn))
891                 return NULL;
892
893         page = pfn_to_page(pfn);
894
895         BUG_ON(PageHighMem(page));
896
897         if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
898                 return NULL;
899
900         if (PageReserved(page) && pfn_is_nosave(pfn))
901                 return NULL;
902
903         return page;
904 }
905
906 /**
907  *      count_data_pages - compute the total number of saveable non-highmem
908  *      pages.
909  */
910
911 unsigned int count_data_pages(void)
912 {
913         struct zone *zone;
914         unsigned long pfn, max_zone_pfn;
915         unsigned int n = 0;
916
917         for_each_zone(zone) {
918                 if (is_highmem(zone))
919                         continue;
920
921                 mark_free_pages(zone);
922                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
923                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
924                         if(saveable_page(pfn))
925                                 n++;
926         }
927         return n;
928 }
929
930 /* This is needed, because copy_page and memcpy are not usable for copying
931  * task structs.
932  */
933 static inline void do_copy_page(long *dst, long *src)
934 {
935         int n;
936
937         for (n = PAGE_SIZE / sizeof(long); n; n--)
938                 *dst++ = *src++;
939 }
940
941 #ifdef CONFIG_HIGHMEM
942 static inline struct page *
943 page_is_saveable(struct zone *zone, unsigned long pfn)
944 {
945         return is_highmem(zone) ?
946                         saveable_highmem_page(pfn) : saveable_page(pfn);
947 }
948
949 static inline void
950 copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
951 {
952         struct page *s_page, *d_page;
953         void *src, *dst;
954
955         s_page = pfn_to_page(src_pfn);
956         d_page = pfn_to_page(dst_pfn);
957         if (PageHighMem(s_page)) {
958                 src = kmap_atomic(s_page, KM_USER0);
959                 dst = kmap_atomic(d_page, KM_USER1);
960                 do_copy_page(dst, src);
961                 kunmap_atomic(src, KM_USER0);
962                 kunmap_atomic(dst, KM_USER1);
963         } else {
964                 src = page_address(s_page);
965                 if (PageHighMem(d_page)) {
966                         /* Page pointed to by src may contain some kernel
967                          * data modified by kmap_atomic()
968                          */
969                         do_copy_page(buffer, src);
970                         dst = kmap_atomic(pfn_to_page(dst_pfn), KM_USER0);
971                         memcpy(dst, buffer, PAGE_SIZE);
972                         kunmap_atomic(dst, KM_USER0);
973                 } else {
974                         dst = page_address(d_page);
975                         do_copy_page(dst, src);
976                 }
977         }
978 }
979 #else
980 #define page_is_saveable(zone, pfn)     saveable_page(pfn)
981
982 static inline void
983 copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
984 {
985         do_copy_page(page_address(pfn_to_page(dst_pfn)),
986                         page_address(pfn_to_page(src_pfn)));
987 }
988 #endif /* CONFIG_HIGHMEM */
989
990 static void
991 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
992 {
993         struct zone *zone;
994         unsigned long pfn;
995
996         for_each_zone(zone) {
997                 unsigned long max_zone_pfn;
998
999                 mark_free_pages(zone);
1000                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1001                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1002                         if (page_is_saveable(zone, pfn))
1003                                 memory_bm_set_bit(orig_bm, pfn);
1004         }
1005         memory_bm_position_reset(orig_bm);
1006         memory_bm_position_reset(copy_bm);
1007         for(;;) {
1008                 pfn = memory_bm_next_pfn(orig_bm);
1009                 if (unlikely(pfn == BM_END_OF_MAP))
1010                         break;
1011                 copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1012         }
1013 }
1014
1015 /* Total number of image pages */
1016 static unsigned int nr_copy_pages;
1017 /* Number of pages needed for saving the original pfns of the image pages */
1018 static unsigned int nr_meta_pages;
1019
1020 /**
1021  *      swsusp_free - free pages allocated for the suspend.
1022  *
1023  *      Suspend pages are alocated before the atomic copy is made, so we
1024  *      need to release them after the resume.
1025  */
1026
1027 void swsusp_free(void)
1028 {
1029         struct zone *zone;
1030         unsigned long pfn, max_zone_pfn;
1031
1032         for_each_zone(zone) {
1033                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1034                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1035                         if (pfn_valid(pfn)) {
1036                                 struct page *page = pfn_to_page(pfn);
1037
1038                                 if (swsusp_page_is_forbidden(page) &&
1039                                     swsusp_page_is_free(page)) {
1040                                         swsusp_unset_page_forbidden(page);
1041                                         swsusp_unset_page_free(page);
1042                                         __free_page(page);
1043                                 }
1044                         }
1045         }
1046         nr_copy_pages = 0;
1047         nr_meta_pages = 0;
1048         restore_pblist = NULL;
1049         buffer = NULL;
1050 }
1051
1052 #ifdef CONFIG_HIGHMEM
1053 /**
1054   *     count_pages_for_highmem - compute the number of non-highmem pages
1055   *     that will be necessary for creating copies of highmem pages.
1056   */
1057
1058 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1059 {
1060         unsigned int free_highmem = count_free_highmem_pages();
1061
1062         if (free_highmem >= nr_highmem)
1063                 nr_highmem = 0;
1064         else
1065                 nr_highmem -= free_highmem;
1066
1067         return nr_highmem;
1068 }
1069 #else
1070 static unsigned int
1071 count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1072 #endif /* CONFIG_HIGHMEM */
1073
1074 /**
1075  *      enough_free_mem - Make sure we have enough free memory for the
1076  *      snapshot image.
1077  */
1078
1079 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1080 {
1081         struct zone *zone;
1082         unsigned int free = 0, meta = 0;
1083
1084         for_each_zone(zone) {
1085                 meta += snapshot_additional_pages(zone);
1086                 if (!is_highmem(zone))
1087                         free += zone_page_state(zone, NR_FREE_PAGES);
1088         }
1089
1090         nr_pages += count_pages_for_highmem(nr_highmem);
1091         pr_debug("PM: Normal pages needed: %u + %u + %u, available pages: %u\n",
1092                 nr_pages, PAGES_FOR_IO, meta, free);
1093
1094         return free > nr_pages + PAGES_FOR_IO + meta;
1095 }
1096
1097 #ifdef CONFIG_HIGHMEM
1098 /**
1099  *      get_highmem_buffer - if there are some highmem pages in the suspend
1100  *      image, we may need the buffer to copy them and/or load their data.
1101  */
1102
1103 static inline int get_highmem_buffer(int safe_needed)
1104 {
1105         buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1106         return buffer ? 0 : -ENOMEM;
1107 }
1108
1109 /**
1110  *      alloc_highmem_image_pages - allocate some highmem pages for the image.
1111  *      Try to allocate as many pages as needed, but if the number of free
1112  *      highmem pages is lesser than that, allocate them all.
1113  */
1114
1115 static inline unsigned int
1116 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1117 {
1118         unsigned int to_alloc = count_free_highmem_pages();
1119
1120         if (to_alloc > nr_highmem)
1121                 to_alloc = nr_highmem;
1122
1123         nr_highmem -= to_alloc;
1124         while (to_alloc-- > 0) {
1125                 struct page *page;
1126
1127                 page = alloc_image_page(__GFP_HIGHMEM);
1128                 memory_bm_set_bit(bm, page_to_pfn(page));
1129         }
1130         return nr_highmem;
1131 }
1132 #else
1133 static inline int get_highmem_buffer(int safe_needed) { return 0; }
1134
1135 static inline unsigned int
1136 alloc_highmem_image_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1137 #endif /* CONFIG_HIGHMEM */
1138
1139 /**
1140  *      swsusp_alloc - allocate memory for the suspend image
1141  *
1142  *      We first try to allocate as many highmem pages as there are
1143  *      saveable highmem pages in the system.  If that fails, we allocate
1144  *      non-highmem pages for the copies of the remaining highmem ones.
1145  *
1146  *      In this approach it is likely that the copies of highmem pages will
1147  *      also be located in the high memory, because of the way in which
1148  *      copy_data_pages() works.
1149  */
1150
1151 static int
1152 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1153                 unsigned int nr_pages, unsigned int nr_highmem)
1154 {
1155         int error;
1156
1157         error = memory_bm_create(orig_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1158         if (error)
1159                 goto Free;
1160
1161         error = memory_bm_create(copy_bm, GFP_ATOMIC | __GFP_COLD, PG_ANY);
1162         if (error)
1163                 goto Free;
1164
1165         if (nr_highmem > 0) {
1166                 error = get_highmem_buffer(PG_ANY);
1167                 if (error)
1168                         goto Free;
1169
1170                 nr_pages += alloc_highmem_image_pages(copy_bm, nr_highmem);
1171         }
1172         while (nr_pages-- > 0) {
1173                 struct page *page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1174
1175                 if (!page)
1176                         goto Free;
1177
1178                 memory_bm_set_bit(copy_bm, page_to_pfn(page));
1179         }
1180         return 0;
1181
1182  Free:
1183         swsusp_free();
1184         return -ENOMEM;
1185 }
1186
1187 /* Memory bitmap used for marking saveable pages (during suspend) or the
1188  * suspend image pages (during resume)
1189  */
1190 static struct memory_bitmap orig_bm;
1191 /* Memory bitmap used on suspend for marking allocated pages that will contain
1192  * the copies of saveable pages.  During resume it is initially used for
1193  * marking the suspend image pages, but then its set bits are duplicated in
1194  * @orig_bm and it is released.  Next, on systems with high memory, it may be
1195  * used for marking "safe" highmem pages, but it has to be reinitialized for
1196  * this purpose.
1197  */
1198 static struct memory_bitmap copy_bm;
1199
1200 asmlinkage int swsusp_save(void)
1201 {
1202         unsigned int nr_pages, nr_highmem;
1203
1204         printk(KERN_INFO "PM: Creating hibernation image: \n");
1205
1206         drain_local_pages(NULL);
1207         nr_pages = count_data_pages();
1208         nr_highmem = count_highmem_pages();
1209         printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1210
1211         if (!enough_free_mem(nr_pages, nr_highmem)) {
1212                 printk(KERN_ERR "PM: Not enough free memory\n");
1213                 return -ENOMEM;
1214         }
1215
1216         if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1217                 printk(KERN_ERR "PM: Memory allocation failed\n");
1218                 return -ENOMEM;
1219         }
1220
1221         /* During allocating of suspend pagedir, new cold pages may appear.
1222          * Kill them.
1223          */
1224         drain_local_pages(NULL);
1225         copy_data_pages(&copy_bm, &orig_bm);
1226
1227         /*
1228          * End of critical section. From now on, we can write to memory,
1229          * but we should not touch disk. This specially means we must _not_
1230          * touch swap space! Except we must write out our image of course.
1231          */
1232
1233         nr_pages += nr_highmem;
1234         nr_copy_pages = nr_pages;
1235         nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1236
1237         printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1238                 nr_pages);
1239
1240         return 0;
1241 }
1242
1243 #ifndef CONFIG_ARCH_HIBERNATION_HEADER
1244 static int init_header_complete(struct swsusp_info *info)
1245 {
1246         memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1247         info->version_code = LINUX_VERSION_CODE;
1248         return 0;
1249 }
1250
1251 static char *check_image_kernel(struct swsusp_info *info)
1252 {
1253         if (info->version_code != LINUX_VERSION_CODE)
1254                 return "kernel version";
1255         if (strcmp(info->uts.sysname,init_utsname()->sysname))
1256                 return "system type";
1257         if (strcmp(info->uts.release,init_utsname()->release))
1258                 return "kernel release";
1259         if (strcmp(info->uts.version,init_utsname()->version))
1260                 return "version";
1261         if (strcmp(info->uts.machine,init_utsname()->machine))
1262                 return "machine";
1263         return NULL;
1264 }
1265 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1266
1267 unsigned long snapshot_get_image_size(void)
1268 {
1269         return nr_copy_pages + nr_meta_pages + 1;
1270 }
1271
1272 static int init_header(struct swsusp_info *info)
1273 {
1274         memset(info, 0, sizeof(struct swsusp_info));
1275         info->num_physpages = num_physpages;
1276         info->image_pages = nr_copy_pages;
1277         info->pages = snapshot_get_image_size();
1278         info->size = info->pages;
1279         info->size <<= PAGE_SHIFT;
1280         return init_header_complete(info);
1281 }
1282
1283 /**
1284  *      pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1285  *      are stored in the array @buf[] (1 page at a time)
1286  */
1287
1288 static inline void
1289 pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1290 {
1291         int j;
1292
1293         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1294                 buf[j] = memory_bm_next_pfn(bm);
1295                 if (unlikely(buf[j] == BM_END_OF_MAP))
1296                         break;
1297         }
1298 }
1299
1300 /**
1301  *      snapshot_read_next - used for reading the system memory snapshot.
1302  *
1303  *      On the first call to it @handle should point to a zeroed
1304  *      snapshot_handle structure.  The structure gets updated and a pointer
1305  *      to it should be passed to this function every next time.
1306  *
1307  *      The @count parameter should contain the number of bytes the caller
1308  *      wants to read from the snapshot.  It must not be zero.
1309  *
1310  *      On success the function returns a positive number.  Then, the caller
1311  *      is allowed to read up to the returned number of bytes from the memory
1312  *      location computed by the data_of() macro.  The number returned
1313  *      may be smaller than @count, but this only happens if the read would
1314  *      cross a page boundary otherwise.
1315  *
1316  *      The function returns 0 to indicate the end of data stream condition,
1317  *      and a negative number is returned on error.  In such cases the
1318  *      structure pointed to by @handle is not updated and should not be used
1319  *      any more.
1320  */
1321
1322 int snapshot_read_next(struct snapshot_handle *handle, size_t count)
1323 {
1324         if (handle->cur > nr_meta_pages + nr_copy_pages)
1325                 return 0;
1326
1327         if (!buffer) {
1328                 /* This makes the buffer be freed by swsusp_free() */
1329                 buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1330                 if (!buffer)
1331                         return -ENOMEM;
1332         }
1333         if (!handle->offset) {
1334                 int error;
1335
1336                 error = init_header((struct swsusp_info *)buffer);
1337                 if (error)
1338                         return error;
1339                 handle->buffer = buffer;
1340                 memory_bm_position_reset(&orig_bm);
1341                 memory_bm_position_reset(&copy_bm);
1342         }
1343         if (handle->prev < handle->cur) {
1344                 if (handle->cur <= nr_meta_pages) {
1345                         memset(buffer, 0, PAGE_SIZE);
1346                         pack_pfns(buffer, &orig_bm);
1347                 } else {
1348                         struct page *page;
1349
1350                         page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1351                         if (PageHighMem(page)) {
1352                                 /* Highmem pages are copied to the buffer,
1353                                  * because we can't return with a kmapped
1354                                  * highmem page (we may not be called again).
1355                                  */
1356                                 void *kaddr;
1357
1358                                 kaddr = kmap_atomic(page, KM_USER0);
1359                                 memcpy(buffer, kaddr, PAGE_SIZE);
1360                                 kunmap_atomic(kaddr, KM_USER0);
1361                                 handle->buffer = buffer;
1362                         } else {
1363                                 handle->buffer = page_address(page);
1364                         }
1365                 }
1366                 handle->prev = handle->cur;
1367         }
1368         handle->buf_offset = handle->cur_offset;
1369         if (handle->cur_offset + count >= PAGE_SIZE) {
1370                 count = PAGE_SIZE - handle->cur_offset;
1371                 handle->cur_offset = 0;
1372                 handle->cur++;
1373         } else {
1374                 handle->cur_offset += count;
1375         }
1376         handle->offset += count;
1377         return count;
1378 }
1379
1380 /**
1381  *      mark_unsafe_pages - mark the pages that cannot be used for storing
1382  *      the image during resume, because they conflict with the pages that
1383  *      had been used before suspend
1384  */
1385
1386 static int mark_unsafe_pages(struct memory_bitmap *bm)
1387 {
1388         struct zone *zone;
1389         unsigned long pfn, max_zone_pfn;
1390
1391         /* Clear page flags */
1392         for_each_zone(zone) {
1393                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1394                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1395                         if (pfn_valid(pfn))
1396                                 swsusp_unset_page_free(pfn_to_page(pfn));
1397         }
1398
1399         /* Mark pages that correspond to the "original" pfns as "unsafe" */
1400         memory_bm_position_reset(bm);
1401         do {
1402                 pfn = memory_bm_next_pfn(bm);
1403                 if (likely(pfn != BM_END_OF_MAP)) {
1404                         if (likely(pfn_valid(pfn)))
1405                                 swsusp_set_page_free(pfn_to_page(pfn));
1406                         else
1407                                 return -EFAULT;
1408                 }
1409         } while (pfn != BM_END_OF_MAP);
1410
1411         allocated_unsafe_pages = 0;
1412
1413         return 0;
1414 }
1415
1416 static void
1417 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1418 {
1419         unsigned long pfn;
1420
1421         memory_bm_position_reset(src);
1422         pfn = memory_bm_next_pfn(src);
1423         while (pfn != BM_END_OF_MAP) {
1424                 memory_bm_set_bit(dst, pfn);
1425                 pfn = memory_bm_next_pfn(src);
1426         }
1427 }
1428
1429 static int check_header(struct swsusp_info *info)
1430 {
1431         char *reason;
1432
1433         reason = check_image_kernel(info);
1434         if (!reason && info->num_physpages != num_physpages)
1435                 reason = "memory size";
1436         if (reason) {
1437                 printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1438                 return -EPERM;
1439         }
1440         return 0;
1441 }
1442
1443 /**
1444  *      load header - check the image header and copy data from it
1445  */
1446
1447 static int
1448 load_header(struct swsusp_info *info)
1449 {
1450         int error;
1451
1452         restore_pblist = NULL;
1453         error = check_header(info);
1454         if (!error) {
1455                 nr_copy_pages = info->image_pages;
1456                 nr_meta_pages = info->pages - info->image_pages - 1;
1457         }
1458         return error;
1459 }
1460
1461 /**
1462  *      unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1463  *      the corresponding bit in the memory bitmap @bm
1464  */
1465
1466 static inline void
1467 unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1468 {
1469         int j;
1470
1471         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1472                 if (unlikely(buf[j] == BM_END_OF_MAP))
1473                         break;
1474
1475                 memory_bm_set_bit(bm, buf[j]);
1476         }
1477 }
1478
1479 /* List of "safe" pages that may be used to store data loaded from the suspend
1480  * image
1481  */
1482 static struct linked_page *safe_pages_list;
1483
1484 #ifdef CONFIG_HIGHMEM
1485 /* struct highmem_pbe is used for creating the list of highmem pages that
1486  * should be restored atomically during the resume from disk, because the page
1487  * frames they have occupied before the suspend are in use.
1488  */
1489 struct highmem_pbe {
1490         struct page *copy_page; /* data is here now */
1491         struct page *orig_page; /* data was here before the suspend */
1492         struct highmem_pbe *next;
1493 };
1494
1495 /* List of highmem PBEs needed for restoring the highmem pages that were
1496  * allocated before the suspend and included in the suspend image, but have
1497  * also been allocated by the "resume" kernel, so their contents cannot be
1498  * written directly to their "original" page frames.
1499  */
1500 static struct highmem_pbe *highmem_pblist;
1501
1502 /**
1503  *      count_highmem_image_pages - compute the number of highmem pages in the
1504  *      suspend image.  The bits in the memory bitmap @bm that correspond to the
1505  *      image pages are assumed to be set.
1506  */
1507
1508 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1509 {
1510         unsigned long pfn;
1511         unsigned int cnt = 0;
1512
1513         memory_bm_position_reset(bm);
1514         pfn = memory_bm_next_pfn(bm);
1515         while (pfn != BM_END_OF_MAP) {
1516                 if (PageHighMem(pfn_to_page(pfn)))
1517                         cnt++;
1518
1519                 pfn = memory_bm_next_pfn(bm);
1520         }
1521         return cnt;
1522 }
1523
1524 /**
1525  *      prepare_highmem_image - try to allocate as many highmem pages as
1526  *      there are highmem image pages (@nr_highmem_p points to the variable
1527  *      containing the number of highmem image pages).  The pages that are
1528  *      "safe" (ie. will not be overwritten when the suspend image is
1529  *      restored) have the corresponding bits set in @bm (it must be
1530  *      unitialized).
1531  *
1532  *      NOTE: This function should not be called if there are no highmem
1533  *      image pages.
1534  */
1535
1536 static unsigned int safe_highmem_pages;
1537
1538 static struct memory_bitmap *safe_highmem_bm;
1539
1540 static int
1541 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1542 {
1543         unsigned int to_alloc;
1544
1545         if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1546                 return -ENOMEM;
1547
1548         if (get_highmem_buffer(PG_SAFE))
1549                 return -ENOMEM;
1550
1551         to_alloc = count_free_highmem_pages();
1552         if (to_alloc > *nr_highmem_p)
1553                 to_alloc = *nr_highmem_p;
1554         else
1555                 *nr_highmem_p = to_alloc;
1556
1557         safe_highmem_pages = 0;
1558         while (to_alloc-- > 0) {
1559                 struct page *page;
1560
1561                 page = alloc_page(__GFP_HIGHMEM);
1562                 if (!swsusp_page_is_free(page)) {
1563                         /* The page is "safe", set its bit the bitmap */
1564                         memory_bm_set_bit(bm, page_to_pfn(page));
1565                         safe_highmem_pages++;
1566                 }
1567                 /* Mark the page as allocated */
1568                 swsusp_set_page_forbidden(page);
1569                 swsusp_set_page_free(page);
1570         }
1571         memory_bm_position_reset(bm);
1572         safe_highmem_bm = bm;
1573         return 0;
1574 }
1575
1576 /**
1577  *      get_highmem_page_buffer - for given highmem image page find the buffer
1578  *      that suspend_write_next() should set for its caller to write to.
1579  *
1580  *      If the page is to be saved to its "original" page frame or a copy of
1581  *      the page is to be made in the highmem, @buffer is returned.  Otherwise,
1582  *      the copy of the page is to be made in normal memory, so the address of
1583  *      the copy is returned.
1584  *
1585  *      If @buffer is returned, the caller of suspend_write_next() will write
1586  *      the page's contents to @buffer, so they will have to be copied to the
1587  *      right location on the next call to suspend_write_next() and it is done
1588  *      with the help of copy_last_highmem_page().  For this purpose, if
1589  *      @buffer is returned, @last_highmem page is set to the page to which
1590  *      the data will have to be copied from @buffer.
1591  */
1592
1593 static struct page *last_highmem_page;
1594
1595 static void *
1596 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1597 {
1598         struct highmem_pbe *pbe;
1599         void *kaddr;
1600
1601         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1602                 /* We have allocated the "original" page frame and we can
1603                  * use it directly to store the loaded page.
1604                  */
1605                 last_highmem_page = page;
1606                 return buffer;
1607         }
1608         /* The "original" page frame has not been allocated and we have to
1609          * use a "safe" page frame to store the loaded page.
1610          */
1611         pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1612         if (!pbe) {
1613                 swsusp_free();
1614                 return NULL;
1615         }
1616         pbe->orig_page = page;
1617         if (safe_highmem_pages > 0) {
1618                 struct page *tmp;
1619
1620                 /* Copy of the page will be stored in high memory */
1621                 kaddr = buffer;
1622                 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1623                 safe_highmem_pages--;
1624                 last_highmem_page = tmp;
1625                 pbe->copy_page = tmp;
1626         } else {
1627                 /* Copy of the page will be stored in normal memory */
1628                 kaddr = safe_pages_list;
1629                 safe_pages_list = safe_pages_list->next;
1630                 pbe->copy_page = virt_to_page(kaddr);
1631         }
1632         pbe->next = highmem_pblist;
1633         highmem_pblist = pbe;
1634         return kaddr;
1635 }
1636
1637 /**
1638  *      copy_last_highmem_page - copy the contents of a highmem image from
1639  *      @buffer, where the caller of snapshot_write_next() has place them,
1640  *      to the right location represented by @last_highmem_page .
1641  */
1642
1643 static void copy_last_highmem_page(void)
1644 {
1645         if (last_highmem_page) {
1646                 void *dst;
1647
1648                 dst = kmap_atomic(last_highmem_page, KM_USER0);
1649                 memcpy(dst, buffer, PAGE_SIZE);
1650                 kunmap_atomic(dst, KM_USER0);
1651                 last_highmem_page = NULL;
1652         }
1653 }
1654
1655 static inline int last_highmem_page_copied(void)
1656 {
1657         return !last_highmem_page;
1658 }
1659
1660 static inline void free_highmem_data(void)
1661 {
1662         if (safe_highmem_bm)
1663                 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
1664
1665         if (buffer)
1666                 free_image_page(buffer, PG_UNSAFE_CLEAR);
1667 }
1668 #else
1669 static inline int get_safe_write_buffer(void) { return 0; }
1670
1671 static unsigned int
1672 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
1673
1674 static inline int
1675 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1676 {
1677         return 0;
1678 }
1679
1680 static inline void *
1681 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1682 {
1683         return NULL;
1684 }
1685
1686 static inline void copy_last_highmem_page(void) {}
1687 static inline int last_highmem_page_copied(void) { return 1; }
1688 static inline void free_highmem_data(void) {}
1689 #endif /* CONFIG_HIGHMEM */
1690
1691 /**
1692  *      prepare_image - use the memory bitmap @bm to mark the pages that will
1693  *      be overwritten in the process of restoring the system memory state
1694  *      from the suspend image ("unsafe" pages) and allocate memory for the
1695  *      image.
1696  *
1697  *      The idea is to allocate a new memory bitmap first and then allocate
1698  *      as many pages as needed for the image data, but not to assign these
1699  *      pages to specific tasks initially.  Instead, we just mark them as
1700  *      allocated and create a lists of "safe" pages that will be used
1701  *      later.  On systems with high memory a list of "safe" highmem pages is
1702  *      also created.
1703  */
1704
1705 #define PBES_PER_LINKED_PAGE    (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
1706
1707 static int
1708 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
1709 {
1710         unsigned int nr_pages, nr_highmem;
1711         struct linked_page *sp_list, *lp;
1712         int error;
1713
1714         /* If there is no highmem, the buffer will not be necessary */
1715         free_image_page(buffer, PG_UNSAFE_CLEAR);
1716         buffer = NULL;
1717
1718         nr_highmem = count_highmem_image_pages(bm);
1719         error = mark_unsafe_pages(bm);
1720         if (error)
1721                 goto Free;
1722
1723         error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
1724         if (error)
1725                 goto Free;
1726
1727         duplicate_memory_bitmap(new_bm, bm);
1728         memory_bm_free(bm, PG_UNSAFE_KEEP);
1729         if (nr_highmem > 0) {
1730                 error = prepare_highmem_image(bm, &nr_highmem);
1731                 if (error)
1732                         goto Free;
1733         }
1734         /* Reserve some safe pages for potential later use.
1735          *
1736          * NOTE: This way we make sure there will be enough safe pages for the
1737          * chain_alloc() in get_buffer().  It is a bit wasteful, but
1738          * nr_copy_pages cannot be greater than 50% of the memory anyway.
1739          */
1740         sp_list = NULL;
1741         /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
1742         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1743         nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
1744         while (nr_pages > 0) {
1745                 lp = get_image_page(GFP_ATOMIC, PG_SAFE);
1746                 if (!lp) {
1747                         error = -ENOMEM;
1748                         goto Free;
1749                 }
1750                 lp->next = sp_list;
1751                 sp_list = lp;
1752                 nr_pages--;
1753         }
1754         /* Preallocate memory for the image */
1755         safe_pages_list = NULL;
1756         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
1757         while (nr_pages > 0) {
1758                 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
1759                 if (!lp) {
1760                         error = -ENOMEM;
1761                         goto Free;
1762                 }
1763                 if (!swsusp_page_is_free(virt_to_page(lp))) {
1764                         /* The page is "safe", add it to the list */
1765                         lp->next = safe_pages_list;
1766                         safe_pages_list = lp;
1767                 }
1768                 /* Mark the page as allocated */
1769                 swsusp_set_page_forbidden(virt_to_page(lp));
1770                 swsusp_set_page_free(virt_to_page(lp));
1771                 nr_pages--;
1772         }
1773         /* Free the reserved safe pages so that chain_alloc() can use them */
1774         while (sp_list) {
1775                 lp = sp_list->next;
1776                 free_image_page(sp_list, PG_UNSAFE_CLEAR);
1777                 sp_list = lp;
1778         }
1779         return 0;
1780
1781  Free:
1782         swsusp_free();
1783         return error;
1784 }
1785
1786 /**
1787  *      get_buffer - compute the address that snapshot_write_next() should
1788  *      set for its caller to write to.
1789  */
1790
1791 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
1792 {
1793         struct pbe *pbe;
1794         struct page *page = pfn_to_page(memory_bm_next_pfn(bm));
1795
1796         if (PageHighMem(page))
1797                 return get_highmem_page_buffer(page, ca);
1798
1799         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
1800                 /* We have allocated the "original" page frame and we can
1801                  * use it directly to store the loaded page.
1802                  */
1803                 return page_address(page);
1804
1805         /* The "original" page frame has not been allocated and we have to
1806          * use a "safe" page frame to store the loaded page.
1807          */
1808         pbe = chain_alloc(ca, sizeof(struct pbe));
1809         if (!pbe) {
1810                 swsusp_free();
1811                 return NULL;
1812         }
1813         pbe->orig_address = page_address(page);
1814         pbe->address = safe_pages_list;
1815         safe_pages_list = safe_pages_list->next;
1816         pbe->next = restore_pblist;
1817         restore_pblist = pbe;
1818         return pbe->address;
1819 }
1820
1821 /**
1822  *      snapshot_write_next - used for writing the system memory snapshot.
1823  *
1824  *      On the first call to it @handle should point to a zeroed
1825  *      snapshot_handle structure.  The structure gets updated and a pointer
1826  *      to it should be passed to this function every next time.
1827  *
1828  *      The @count parameter should contain the number of bytes the caller
1829  *      wants to write to the image.  It must not be zero.
1830  *
1831  *      On success the function returns a positive number.  Then, the caller
1832  *      is allowed to write up to the returned number of bytes to the memory
1833  *      location computed by the data_of() macro.  The number returned
1834  *      may be smaller than @count, but this only happens if the write would
1835  *      cross a page boundary otherwise.
1836  *
1837  *      The function returns 0 to indicate the "end of file" condition,
1838  *      and a negative number is returned on error.  In such cases the
1839  *      structure pointed to by @handle is not updated and should not be used
1840  *      any more.
1841  */
1842
1843 int snapshot_write_next(struct snapshot_handle *handle, size_t count)
1844 {
1845         static struct chain_allocator ca;
1846         int error = 0;
1847
1848         /* Check if we have already loaded the entire image */
1849         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages)
1850                 return 0;
1851
1852         if (handle->offset == 0) {
1853                 if (!buffer)
1854                         /* This makes the buffer be freed by swsusp_free() */
1855                         buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1856
1857                 if (!buffer)
1858                         return -ENOMEM;
1859
1860                 handle->buffer = buffer;
1861         }
1862         handle->sync_read = 1;
1863         if (handle->prev < handle->cur) {
1864                 if (handle->prev == 0) {
1865                         error = load_header(buffer);
1866                         if (error)
1867                                 return error;
1868
1869                         error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
1870                         if (error)
1871                                 return error;
1872
1873                 } else if (handle->prev <= nr_meta_pages) {
1874                         unpack_orig_pfns(buffer, &copy_bm);
1875                         if (handle->prev == nr_meta_pages) {
1876                                 error = prepare_image(&orig_bm, &copy_bm);
1877                                 if (error)
1878                                         return error;
1879
1880                                 chain_init(&ca, GFP_ATOMIC, PG_SAFE);
1881                                 memory_bm_position_reset(&orig_bm);
1882                                 restore_pblist = NULL;
1883                                 handle->buffer = get_buffer(&orig_bm, &ca);
1884                                 handle->sync_read = 0;
1885                                 if (!handle->buffer)
1886                                         return -ENOMEM;
1887                         }
1888                 } else {
1889                         copy_last_highmem_page();
1890                         handle->buffer = get_buffer(&orig_bm, &ca);
1891                         if (handle->buffer != buffer)
1892                                 handle->sync_read = 0;
1893                 }
1894                 handle->prev = handle->cur;
1895         }
1896         handle->buf_offset = handle->cur_offset;
1897         if (handle->cur_offset + count >= PAGE_SIZE) {
1898                 count = PAGE_SIZE - handle->cur_offset;
1899                 handle->cur_offset = 0;
1900                 handle->cur++;
1901         } else {
1902                 handle->cur_offset += count;
1903         }
1904         handle->offset += count;
1905         return count;
1906 }
1907
1908 /**
1909  *      snapshot_write_finalize - must be called after the last call to
1910  *      snapshot_write_next() in case the last page in the image happens
1911  *      to be a highmem page and its contents should be stored in the
1912  *      highmem.  Additionally, it releases the memory that will not be
1913  *      used any more.
1914  */
1915
1916 void snapshot_write_finalize(struct snapshot_handle *handle)
1917 {
1918         copy_last_highmem_page();
1919         /* Free only if we have loaded the image entirely */
1920         if (handle->prev && handle->cur > nr_meta_pages + nr_copy_pages) {
1921                 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
1922                 free_highmem_data();
1923         }
1924 }
1925
1926 int snapshot_image_loaded(struct snapshot_handle *handle)
1927 {
1928         return !(!nr_copy_pages || !last_highmem_page_copied() ||
1929                         handle->cur <= nr_meta_pages + nr_copy_pages);
1930 }
1931
1932 #ifdef CONFIG_HIGHMEM
1933 /* Assumes that @buf is ready and points to a "safe" page */
1934 static inline void
1935 swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
1936 {
1937         void *kaddr1, *kaddr2;
1938
1939         kaddr1 = kmap_atomic(p1, KM_USER0);
1940         kaddr2 = kmap_atomic(p2, KM_USER1);
1941         memcpy(buf, kaddr1, PAGE_SIZE);
1942         memcpy(kaddr1, kaddr2, PAGE_SIZE);
1943         memcpy(kaddr2, buf, PAGE_SIZE);
1944         kunmap_atomic(kaddr1, KM_USER0);
1945         kunmap_atomic(kaddr2, KM_USER1);
1946 }
1947
1948 /**
1949  *      restore_highmem - for each highmem page that was allocated before
1950  *      the suspend and included in the suspend image, and also has been
1951  *      allocated by the "resume" kernel swap its current (ie. "before
1952  *      resume") contents with the previous (ie. "before suspend") one.
1953  *
1954  *      If the resume eventually fails, we can call this function once
1955  *      again and restore the "before resume" highmem state.
1956  */
1957
1958 int restore_highmem(void)
1959 {
1960         struct highmem_pbe *pbe = highmem_pblist;
1961         void *buf;
1962
1963         if (!pbe)
1964                 return 0;
1965
1966         buf = get_image_page(GFP_ATOMIC, PG_SAFE);
1967         if (!buf)
1968                 return -ENOMEM;
1969
1970         while (pbe) {
1971                 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
1972                 pbe = pbe->next;
1973         }
1974         free_image_page(buf, PG_UNSAFE_CLEAR);
1975         return 0;
1976 }
1977 #endif /* CONFIG_HIGHMEM */