PM / Hibernate: Add sysfs knob to control size of memory for drivers
[linux-2.6.git] / kernel / power / snapshot.c
1 /*
2  * linux/kernel/power/snapshot.c
3  *
4  * This file provides system snapshot/restore functionality for swsusp.
5  *
6  * Copyright (C) 1998-2005 Pavel Machek <pavel@ucw.cz>
7  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
8  *
9  * This file is released under the GPLv2.
10  *
11  */
12
13 #include <linux/version.h>
14 #include <linux/module.h>
15 #include <linux/mm.h>
16 #include <linux/suspend.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/spinlock.h>
20 #include <linux/kernel.h>
21 #include <linux/pm.h>
22 #include <linux/device.h>
23 #include <linux/init.h>
24 #include <linux/bootmem.h>
25 #include <linux/syscalls.h>
26 #include <linux/console.h>
27 #include <linux/highmem.h>
28 #include <linux/list.h>
29 #include <linux/slab.h>
30
31 #include <asm/uaccess.h>
32 #include <asm/mmu_context.h>
33 #include <asm/pgtable.h>
34 #include <asm/tlbflush.h>
35 #include <asm/io.h>
36
37 #include "power.h"
38
39 static int swsusp_page_is_free(struct page *);
40 static void swsusp_set_page_forbidden(struct page *);
41 static void swsusp_unset_page_forbidden(struct page *);
42
43 /*
44  * Number of bytes to reserve for memory allocations made by device drivers
45  * from their ->freeze() and ->freeze_noirq() callbacks so that they don't
46  * cause image creation to fail (tunable via /sys/power/reserved_size).
47  */
48 unsigned long reserved_size;
49
50 void __init hibernate_reserved_size_init(void)
51 {
52         reserved_size = SPARE_PAGES * PAGE_SIZE;
53 }
54
55 /*
56  * Preferred image size in bytes (tunable via /sys/power/image_size).
57  * When it is set to N, the image creating code will do its best to
58  * ensure the image size will not exceed N bytes, but if that is
59  * impossible, it will try to create the smallest image possible.
60  */
61 unsigned long image_size;
62
63 void __init hibernate_image_size_init(void)
64 {
65         image_size = (totalram_pages / 3) * PAGE_SIZE;
66 }
67
68 /* List of PBEs needed for restoring the pages that were allocated before
69  * the suspend and included in the suspend image, but have also been
70  * allocated by the "resume" kernel, so their contents cannot be written
71  * directly to their "original" page frames.
72  */
73 struct pbe *restore_pblist;
74
75 /* Pointer to an auxiliary buffer (1 page) */
76 static void *buffer;
77
78 /**
79  *      @safe_needed - on resume, for storing the PBE list and the image,
80  *      we can only use memory pages that do not conflict with the pages
81  *      used before suspend.  The unsafe pages have PageNosaveFree set
82  *      and we count them using unsafe_pages.
83  *
84  *      Each allocated image page is marked as PageNosave and PageNosaveFree
85  *      so that swsusp_free() can release it.
86  */
87
88 #define PG_ANY          0
89 #define PG_SAFE         1
90 #define PG_UNSAFE_CLEAR 1
91 #define PG_UNSAFE_KEEP  0
92
93 static unsigned int allocated_unsafe_pages;
94
95 static void *get_image_page(gfp_t gfp_mask, int safe_needed)
96 {
97         void *res;
98
99         res = (void *)get_zeroed_page(gfp_mask);
100         if (safe_needed)
101                 while (res && swsusp_page_is_free(virt_to_page(res))) {
102                         /* The page is unsafe, mark it for swsusp_free() */
103                         swsusp_set_page_forbidden(virt_to_page(res));
104                         allocated_unsafe_pages++;
105                         res = (void *)get_zeroed_page(gfp_mask);
106                 }
107         if (res) {
108                 swsusp_set_page_forbidden(virt_to_page(res));
109                 swsusp_set_page_free(virt_to_page(res));
110         }
111         return res;
112 }
113
114 unsigned long get_safe_page(gfp_t gfp_mask)
115 {
116         return (unsigned long)get_image_page(gfp_mask, PG_SAFE);
117 }
118
119 static struct page *alloc_image_page(gfp_t gfp_mask)
120 {
121         struct page *page;
122
123         page = alloc_page(gfp_mask);
124         if (page) {
125                 swsusp_set_page_forbidden(page);
126                 swsusp_set_page_free(page);
127         }
128         return page;
129 }
130
131 /**
132  *      free_image_page - free page represented by @addr, allocated with
133  *      get_image_page (page flags set by it must be cleared)
134  */
135
136 static inline void free_image_page(void *addr, int clear_nosave_free)
137 {
138         struct page *page;
139
140         BUG_ON(!virt_addr_valid(addr));
141
142         page = virt_to_page(addr);
143
144         swsusp_unset_page_forbidden(page);
145         if (clear_nosave_free)
146                 swsusp_unset_page_free(page);
147
148         __free_page(page);
149 }
150
151 /* struct linked_page is used to build chains of pages */
152
153 #define LINKED_PAGE_DATA_SIZE   (PAGE_SIZE - sizeof(void *))
154
155 struct linked_page {
156         struct linked_page *next;
157         char data[LINKED_PAGE_DATA_SIZE];
158 } __attribute__((packed));
159
160 static inline void
161 free_list_of_pages(struct linked_page *list, int clear_page_nosave)
162 {
163         while (list) {
164                 struct linked_page *lp = list->next;
165
166                 free_image_page(list, clear_page_nosave);
167                 list = lp;
168         }
169 }
170
171 /**
172   *     struct chain_allocator is used for allocating small objects out of
173   *     a linked list of pages called 'the chain'.
174   *
175   *     The chain grows each time when there is no room for a new object in
176   *     the current page.  The allocated objects cannot be freed individually.
177   *     It is only possible to free them all at once, by freeing the entire
178   *     chain.
179   *
180   *     NOTE: The chain allocator may be inefficient if the allocated objects
181   *     are not much smaller than PAGE_SIZE.
182   */
183
184 struct chain_allocator {
185         struct linked_page *chain;      /* the chain */
186         unsigned int used_space;        /* total size of objects allocated out
187                                          * of the current page
188                                          */
189         gfp_t gfp_mask;         /* mask for allocating pages */
190         int safe_needed;        /* if set, only "safe" pages are allocated */
191 };
192
193 static void
194 chain_init(struct chain_allocator *ca, gfp_t gfp_mask, int safe_needed)
195 {
196         ca->chain = NULL;
197         ca->used_space = LINKED_PAGE_DATA_SIZE;
198         ca->gfp_mask = gfp_mask;
199         ca->safe_needed = safe_needed;
200 }
201
202 static void *chain_alloc(struct chain_allocator *ca, unsigned int size)
203 {
204         void *ret;
205
206         if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
207                 struct linked_page *lp;
208
209                 lp = get_image_page(ca->gfp_mask, ca->safe_needed);
210                 if (!lp)
211                         return NULL;
212
213                 lp->next = ca->chain;
214                 ca->chain = lp;
215                 ca->used_space = 0;
216         }
217         ret = ca->chain->data + ca->used_space;
218         ca->used_space += size;
219         return ret;
220 }
221
222 /**
223  *      Data types related to memory bitmaps.
224  *
225  *      Memory bitmap is a structure consiting of many linked lists of
226  *      objects.  The main list's elements are of type struct zone_bitmap
227  *      and each of them corresonds to one zone.  For each zone bitmap
228  *      object there is a list of objects of type struct bm_block that
229  *      represent each blocks of bitmap in which information is stored.
230  *
231  *      struct memory_bitmap contains a pointer to the main list of zone
232  *      bitmap objects, a struct bm_position used for browsing the bitmap,
233  *      and a pointer to the list of pages used for allocating all of the
234  *      zone bitmap objects and bitmap block objects.
235  *
236  *      NOTE: It has to be possible to lay out the bitmap in memory
237  *      using only allocations of order 0.  Additionally, the bitmap is
238  *      designed to work with arbitrary number of zones (this is over the
239  *      top for now, but let's avoid making unnecessary assumptions ;-).
240  *
241  *      struct zone_bitmap contains a pointer to a list of bitmap block
242  *      objects and a pointer to the bitmap block object that has been
243  *      most recently used for setting bits.  Additionally, it contains the
244  *      pfns that correspond to the start and end of the represented zone.
245  *
246  *      struct bm_block contains a pointer to the memory page in which
247  *      information is stored (in the form of a block of bitmap)
248  *      It also contains the pfns that correspond to the start and end of
249  *      the represented memory area.
250  */
251
252 #define BM_END_OF_MAP   (~0UL)
253
254 #define BM_BITS_PER_BLOCK       (PAGE_SIZE * BITS_PER_BYTE)
255
256 struct bm_block {
257         struct list_head hook;  /* hook into a list of bitmap blocks */
258         unsigned long start_pfn;        /* pfn represented by the first bit */
259         unsigned long end_pfn;  /* pfn represented by the last bit plus 1 */
260         unsigned long *data;    /* bitmap representing pages */
261 };
262
263 static inline unsigned long bm_block_bits(struct bm_block *bb)
264 {
265         return bb->end_pfn - bb->start_pfn;
266 }
267
268 /* strcut bm_position is used for browsing memory bitmaps */
269
270 struct bm_position {
271         struct bm_block *block;
272         int bit;
273 };
274
275 struct memory_bitmap {
276         struct list_head blocks;        /* list of bitmap blocks */
277         struct linked_page *p_list;     /* list of pages used to store zone
278                                          * bitmap objects and bitmap block
279                                          * objects
280                                          */
281         struct bm_position cur; /* most recently used bit position */
282 };
283
284 /* Functions that operate on memory bitmaps */
285
286 static void memory_bm_position_reset(struct memory_bitmap *bm)
287 {
288         bm->cur.block = list_entry(bm->blocks.next, struct bm_block, hook);
289         bm->cur.bit = 0;
290 }
291
292 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
293
294 /**
295  *      create_bm_block_list - create a list of block bitmap objects
296  *      @pages - number of pages to track
297  *      @list - list to put the allocated blocks into
298  *      @ca - chain allocator to be used for allocating memory
299  */
300 static int create_bm_block_list(unsigned long pages,
301                                 struct list_head *list,
302                                 struct chain_allocator *ca)
303 {
304         unsigned int nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
305
306         while (nr_blocks-- > 0) {
307                 struct bm_block *bb;
308
309                 bb = chain_alloc(ca, sizeof(struct bm_block));
310                 if (!bb)
311                         return -ENOMEM;
312                 list_add(&bb->hook, list);
313         }
314
315         return 0;
316 }
317
318 struct mem_extent {
319         struct list_head hook;
320         unsigned long start;
321         unsigned long end;
322 };
323
324 /**
325  *      free_mem_extents - free a list of memory extents
326  *      @list - list of extents to empty
327  */
328 static void free_mem_extents(struct list_head *list)
329 {
330         struct mem_extent *ext, *aux;
331
332         list_for_each_entry_safe(ext, aux, list, hook) {
333                 list_del(&ext->hook);
334                 kfree(ext);
335         }
336 }
337
338 /**
339  *      create_mem_extents - create a list of memory extents representing
340  *                           contiguous ranges of PFNs
341  *      @list - list to put the extents into
342  *      @gfp_mask - mask to use for memory allocations
343  */
344 static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
345 {
346         struct zone *zone;
347
348         INIT_LIST_HEAD(list);
349
350         for_each_populated_zone(zone) {
351                 unsigned long zone_start, zone_end;
352                 struct mem_extent *ext, *cur, *aux;
353
354                 zone_start = zone->zone_start_pfn;
355                 zone_end = zone->zone_start_pfn + zone->spanned_pages;
356
357                 list_for_each_entry(ext, list, hook)
358                         if (zone_start <= ext->end)
359                                 break;
360
361                 if (&ext->hook == list || zone_end < ext->start) {
362                         /* New extent is necessary */
363                         struct mem_extent *new_ext;
364
365                         new_ext = kzalloc(sizeof(struct mem_extent), gfp_mask);
366                         if (!new_ext) {
367                                 free_mem_extents(list);
368                                 return -ENOMEM;
369                         }
370                         new_ext->start = zone_start;
371                         new_ext->end = zone_end;
372                         list_add_tail(&new_ext->hook, &ext->hook);
373                         continue;
374                 }
375
376                 /* Merge this zone's range of PFNs with the existing one */
377                 if (zone_start < ext->start)
378                         ext->start = zone_start;
379                 if (zone_end > ext->end)
380                         ext->end = zone_end;
381
382                 /* More merging may be possible */
383                 cur = ext;
384                 list_for_each_entry_safe_continue(cur, aux, list, hook) {
385                         if (zone_end < cur->start)
386                                 break;
387                         if (zone_end < cur->end)
388                                 ext->end = cur->end;
389                         list_del(&cur->hook);
390                         kfree(cur);
391                 }
392         }
393
394         return 0;
395 }
396
397 /**
398   *     memory_bm_create - allocate memory for a memory bitmap
399   */
400 static int
401 memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask, int safe_needed)
402 {
403         struct chain_allocator ca;
404         struct list_head mem_extents;
405         struct mem_extent *ext;
406         int error;
407
408         chain_init(&ca, gfp_mask, safe_needed);
409         INIT_LIST_HEAD(&bm->blocks);
410
411         error = create_mem_extents(&mem_extents, gfp_mask);
412         if (error)
413                 return error;
414
415         list_for_each_entry(ext, &mem_extents, hook) {
416                 struct bm_block *bb;
417                 unsigned long pfn = ext->start;
418                 unsigned long pages = ext->end - ext->start;
419
420                 bb = list_entry(bm->blocks.prev, struct bm_block, hook);
421
422                 error = create_bm_block_list(pages, bm->blocks.prev, &ca);
423                 if (error)
424                         goto Error;
425
426                 list_for_each_entry_continue(bb, &bm->blocks, hook) {
427                         bb->data = get_image_page(gfp_mask, safe_needed);
428                         if (!bb->data) {
429                                 error = -ENOMEM;
430                                 goto Error;
431                         }
432
433                         bb->start_pfn = pfn;
434                         if (pages >= BM_BITS_PER_BLOCK) {
435                                 pfn += BM_BITS_PER_BLOCK;
436                                 pages -= BM_BITS_PER_BLOCK;
437                         } else {
438                                 /* This is executed only once in the loop */
439                                 pfn += pages;
440                         }
441                         bb->end_pfn = pfn;
442                 }
443         }
444
445         bm->p_list = ca.chain;
446         memory_bm_position_reset(bm);
447  Exit:
448         free_mem_extents(&mem_extents);
449         return error;
450
451  Error:
452         bm->p_list = ca.chain;
453         memory_bm_free(bm, PG_UNSAFE_CLEAR);
454         goto Exit;
455 }
456
457 /**
458   *     memory_bm_free - free memory occupied by the memory bitmap @bm
459   */
460 static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
461 {
462         struct bm_block *bb;
463
464         list_for_each_entry(bb, &bm->blocks, hook)
465                 if (bb->data)
466                         free_image_page(bb->data, clear_nosave_free);
467
468         free_list_of_pages(bm->p_list, clear_nosave_free);
469
470         INIT_LIST_HEAD(&bm->blocks);
471 }
472
473 /**
474  *      memory_bm_find_bit - find the bit in the bitmap @bm that corresponds
475  *      to given pfn.  The cur_zone_bm member of @bm and the cur_block member
476  *      of @bm->cur_zone_bm are updated.
477  */
478 static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
479                                 void **addr, unsigned int *bit_nr)
480 {
481         struct bm_block *bb;
482
483         /*
484          * Check if the pfn corresponds to the current bitmap block and find
485          * the block where it fits if this is not the case.
486          */
487         bb = bm->cur.block;
488         if (pfn < bb->start_pfn)
489                 list_for_each_entry_continue_reverse(bb, &bm->blocks, hook)
490                         if (pfn >= bb->start_pfn)
491                                 break;
492
493         if (pfn >= bb->end_pfn)
494                 list_for_each_entry_continue(bb, &bm->blocks, hook)
495                         if (pfn >= bb->start_pfn && pfn < bb->end_pfn)
496                                 break;
497
498         if (&bb->hook == &bm->blocks)
499                 return -EFAULT;
500
501         /* The block has been found */
502         bm->cur.block = bb;
503         pfn -= bb->start_pfn;
504         bm->cur.bit = pfn + 1;
505         *bit_nr = pfn;
506         *addr = bb->data;
507         return 0;
508 }
509
510 static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
511 {
512         void *addr;
513         unsigned int bit;
514         int error;
515
516         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
517         BUG_ON(error);
518         set_bit(bit, addr);
519 }
520
521 static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
522 {
523         void *addr;
524         unsigned int bit;
525         int error;
526
527         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
528         if (!error)
529                 set_bit(bit, addr);
530         return error;
531 }
532
533 static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
534 {
535         void *addr;
536         unsigned int bit;
537         int error;
538
539         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
540         BUG_ON(error);
541         clear_bit(bit, addr);
542 }
543
544 static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
545 {
546         void *addr;
547         unsigned int bit;
548         int error;
549
550         error = memory_bm_find_bit(bm, pfn, &addr, &bit);
551         BUG_ON(error);
552         return test_bit(bit, addr);
553 }
554
555 static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
556 {
557         void *addr;
558         unsigned int bit;
559
560         return !memory_bm_find_bit(bm, pfn, &addr, &bit);
561 }
562
563 /**
564  *      memory_bm_next_pfn - find the pfn that corresponds to the next set bit
565  *      in the bitmap @bm.  If the pfn cannot be found, BM_END_OF_MAP is
566  *      returned.
567  *
568  *      It is required to run memory_bm_position_reset() before the first call to
569  *      this function.
570  */
571
572 static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
573 {
574         struct bm_block *bb;
575         int bit;
576
577         bb = bm->cur.block;
578         do {
579                 bit = bm->cur.bit;
580                 bit = find_next_bit(bb->data, bm_block_bits(bb), bit);
581                 if (bit < bm_block_bits(bb))
582                         goto Return_pfn;
583
584                 bb = list_entry(bb->hook.next, struct bm_block, hook);
585                 bm->cur.block = bb;
586                 bm->cur.bit = 0;
587         } while (&bb->hook != &bm->blocks);
588
589         memory_bm_position_reset(bm);
590         return BM_END_OF_MAP;
591
592  Return_pfn:
593         bm->cur.bit = bit + 1;
594         return bb->start_pfn + bit;
595 }
596
597 /**
598  *      This structure represents a range of page frames the contents of which
599  *      should not be saved during the suspend.
600  */
601
602 struct nosave_region {
603         struct list_head list;
604         unsigned long start_pfn;
605         unsigned long end_pfn;
606 };
607
608 static LIST_HEAD(nosave_regions);
609
610 /**
611  *      register_nosave_region - register a range of page frames the contents
612  *      of which should not be saved during the suspend (to be used in the early
613  *      initialization code)
614  */
615
616 void __init
617 __register_nosave_region(unsigned long start_pfn, unsigned long end_pfn,
618                          int use_kmalloc)
619 {
620         struct nosave_region *region;
621
622         if (start_pfn >= end_pfn)
623                 return;
624
625         if (!list_empty(&nosave_regions)) {
626                 /* Try to extend the previous region (they should be sorted) */
627                 region = list_entry(nosave_regions.prev,
628                                         struct nosave_region, list);
629                 if (region->end_pfn == start_pfn) {
630                         region->end_pfn = end_pfn;
631                         goto Report;
632                 }
633         }
634         if (use_kmalloc) {
635                 /* during init, this shouldn't fail */
636                 region = kmalloc(sizeof(struct nosave_region), GFP_KERNEL);
637                 BUG_ON(!region);
638         } else
639                 /* This allocation cannot fail */
640                 region = alloc_bootmem(sizeof(struct nosave_region));
641         region->start_pfn = start_pfn;
642         region->end_pfn = end_pfn;
643         list_add_tail(&region->list, &nosave_regions);
644  Report:
645         printk(KERN_INFO "PM: Registered nosave memory: %016lx - %016lx\n",
646                 start_pfn << PAGE_SHIFT, end_pfn << PAGE_SHIFT);
647 }
648
649 /*
650  * Set bits in this map correspond to the page frames the contents of which
651  * should not be saved during the suspend.
652  */
653 static struct memory_bitmap *forbidden_pages_map;
654
655 /* Set bits in this map correspond to free page frames. */
656 static struct memory_bitmap *free_pages_map;
657
658 /*
659  * Each page frame allocated for creating the image is marked by setting the
660  * corresponding bits in forbidden_pages_map and free_pages_map simultaneously
661  */
662
663 void swsusp_set_page_free(struct page *page)
664 {
665         if (free_pages_map)
666                 memory_bm_set_bit(free_pages_map, page_to_pfn(page));
667 }
668
669 static int swsusp_page_is_free(struct page *page)
670 {
671         return free_pages_map ?
672                 memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
673 }
674
675 void swsusp_unset_page_free(struct page *page)
676 {
677         if (free_pages_map)
678                 memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
679 }
680
681 static void swsusp_set_page_forbidden(struct page *page)
682 {
683         if (forbidden_pages_map)
684                 memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
685 }
686
687 int swsusp_page_is_forbidden(struct page *page)
688 {
689         return forbidden_pages_map ?
690                 memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
691 }
692
693 static void swsusp_unset_page_forbidden(struct page *page)
694 {
695         if (forbidden_pages_map)
696                 memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
697 }
698
699 /**
700  *      mark_nosave_pages - set bits corresponding to the page frames the
701  *      contents of which should not be saved in a given bitmap.
702  */
703
704 static void mark_nosave_pages(struct memory_bitmap *bm)
705 {
706         struct nosave_region *region;
707
708         if (list_empty(&nosave_regions))
709                 return;
710
711         list_for_each_entry(region, &nosave_regions, list) {
712                 unsigned long pfn;
713
714                 pr_debug("PM: Marking nosave pages: %016lx - %016lx\n",
715                                 region->start_pfn << PAGE_SHIFT,
716                                 region->end_pfn << PAGE_SHIFT);
717
718                 for (pfn = region->start_pfn; pfn < region->end_pfn; pfn++)
719                         if (pfn_valid(pfn)) {
720                                 /*
721                                  * It is safe to ignore the result of
722                                  * mem_bm_set_bit_check() here, since we won't
723                                  * touch the PFNs for which the error is
724                                  * returned anyway.
725                                  */
726                                 mem_bm_set_bit_check(bm, pfn);
727                         }
728         }
729 }
730
731 /**
732  *      create_basic_memory_bitmaps - create bitmaps needed for marking page
733  *      frames that should not be saved and free page frames.  The pointers
734  *      forbidden_pages_map and free_pages_map are only modified if everything
735  *      goes well, because we don't want the bits to be used before both bitmaps
736  *      are set up.
737  */
738
739 int create_basic_memory_bitmaps(void)
740 {
741         struct memory_bitmap *bm1, *bm2;
742         int error = 0;
743
744         BUG_ON(forbidden_pages_map || free_pages_map);
745
746         bm1 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
747         if (!bm1)
748                 return -ENOMEM;
749
750         error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
751         if (error)
752                 goto Free_first_object;
753
754         bm2 = kzalloc(sizeof(struct memory_bitmap), GFP_KERNEL);
755         if (!bm2)
756                 goto Free_first_bitmap;
757
758         error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
759         if (error)
760                 goto Free_second_object;
761
762         forbidden_pages_map = bm1;
763         free_pages_map = bm2;
764         mark_nosave_pages(forbidden_pages_map);
765
766         pr_debug("PM: Basic memory bitmaps created\n");
767
768         return 0;
769
770  Free_second_object:
771         kfree(bm2);
772  Free_first_bitmap:
773         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
774  Free_first_object:
775         kfree(bm1);
776         return -ENOMEM;
777 }
778
779 /**
780  *      free_basic_memory_bitmaps - free memory bitmaps allocated by
781  *      create_basic_memory_bitmaps().  The auxiliary pointers are necessary
782  *      so that the bitmaps themselves are not referred to while they are being
783  *      freed.
784  */
785
786 void free_basic_memory_bitmaps(void)
787 {
788         struct memory_bitmap *bm1, *bm2;
789
790         BUG_ON(!(forbidden_pages_map && free_pages_map));
791
792         bm1 = forbidden_pages_map;
793         bm2 = free_pages_map;
794         forbidden_pages_map = NULL;
795         free_pages_map = NULL;
796         memory_bm_free(bm1, PG_UNSAFE_CLEAR);
797         kfree(bm1);
798         memory_bm_free(bm2, PG_UNSAFE_CLEAR);
799         kfree(bm2);
800
801         pr_debug("PM: Basic memory bitmaps freed\n");
802 }
803
804 /**
805  *      snapshot_additional_pages - estimate the number of additional pages
806  *      be needed for setting up the suspend image data structures for given
807  *      zone (usually the returned value is greater than the exact number)
808  */
809
810 unsigned int snapshot_additional_pages(struct zone *zone)
811 {
812         unsigned int res;
813
814         res = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
815         res += DIV_ROUND_UP(res * sizeof(struct bm_block), PAGE_SIZE);
816         return 2 * res;
817 }
818
819 #ifdef CONFIG_HIGHMEM
820 /**
821  *      count_free_highmem_pages - compute the total number of free highmem
822  *      pages, system-wide.
823  */
824
825 static unsigned int count_free_highmem_pages(void)
826 {
827         struct zone *zone;
828         unsigned int cnt = 0;
829
830         for_each_populated_zone(zone)
831                 if (is_highmem(zone))
832                         cnt += zone_page_state(zone, NR_FREE_PAGES);
833
834         return cnt;
835 }
836
837 /**
838  *      saveable_highmem_page - Determine whether a highmem page should be
839  *      included in the suspend image.
840  *
841  *      We should save the page if it isn't Nosave or NosaveFree, or Reserved,
842  *      and it isn't a part of a free chunk of pages.
843  */
844 static struct page *saveable_highmem_page(struct zone *zone, unsigned long pfn)
845 {
846         struct page *page;
847
848         if (!pfn_valid(pfn))
849                 return NULL;
850
851         page = pfn_to_page(pfn);
852         if (page_zone(page) != zone)
853                 return NULL;
854
855         BUG_ON(!PageHighMem(page));
856
857         if (swsusp_page_is_forbidden(page) ||  swsusp_page_is_free(page) ||
858             PageReserved(page))
859                 return NULL;
860
861         return page;
862 }
863
864 /**
865  *      count_highmem_pages - compute the total number of saveable highmem
866  *      pages.
867  */
868
869 static unsigned int count_highmem_pages(void)
870 {
871         struct zone *zone;
872         unsigned int n = 0;
873
874         for_each_populated_zone(zone) {
875                 unsigned long pfn, max_zone_pfn;
876
877                 if (!is_highmem(zone))
878                         continue;
879
880                 mark_free_pages(zone);
881                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
882                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
883                         if (saveable_highmem_page(zone, pfn))
884                                 n++;
885         }
886         return n;
887 }
888 #else
889 static inline void *saveable_highmem_page(struct zone *z, unsigned long p)
890 {
891         return NULL;
892 }
893 #endif /* CONFIG_HIGHMEM */
894
895 /**
896  *      saveable_page - Determine whether a non-highmem page should be included
897  *      in the suspend image.
898  *
899  *      We should save the page if it isn't Nosave, and is not in the range
900  *      of pages statically defined as 'unsaveable', and it isn't a part of
901  *      a free chunk of pages.
902  */
903 static struct page *saveable_page(struct zone *zone, unsigned long pfn)
904 {
905         struct page *page;
906
907         if (!pfn_valid(pfn))
908                 return NULL;
909
910         page = pfn_to_page(pfn);
911         if (page_zone(page) != zone)
912                 return NULL;
913
914         BUG_ON(PageHighMem(page));
915
916         if (swsusp_page_is_forbidden(page) || swsusp_page_is_free(page))
917                 return NULL;
918
919         if (PageReserved(page)
920             && (!kernel_page_present(page) || pfn_is_nosave(pfn)))
921                 return NULL;
922
923         return page;
924 }
925
926 /**
927  *      count_data_pages - compute the total number of saveable non-highmem
928  *      pages.
929  */
930
931 static unsigned int count_data_pages(void)
932 {
933         struct zone *zone;
934         unsigned long pfn, max_zone_pfn;
935         unsigned int n = 0;
936
937         for_each_populated_zone(zone) {
938                 if (is_highmem(zone))
939                         continue;
940
941                 mark_free_pages(zone);
942                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
943                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
944                         if (saveable_page(zone, pfn))
945                                 n++;
946         }
947         return n;
948 }
949
950 /* This is needed, because copy_page and memcpy are not usable for copying
951  * task structs.
952  */
953 static inline void do_copy_page(long *dst, long *src)
954 {
955         int n;
956
957         for (n = PAGE_SIZE / sizeof(long); n; n--)
958                 *dst++ = *src++;
959 }
960
961
962 /**
963  *      safe_copy_page - check if the page we are going to copy is marked as
964  *              present in the kernel page tables (this always is the case if
965  *              CONFIG_DEBUG_PAGEALLOC is not set and in that case
966  *              kernel_page_present() always returns 'true').
967  */
968 static void safe_copy_page(void *dst, struct page *s_page)
969 {
970         if (kernel_page_present(s_page)) {
971                 do_copy_page(dst, page_address(s_page));
972         } else {
973                 kernel_map_pages(s_page, 1, 1);
974                 do_copy_page(dst, page_address(s_page));
975                 kernel_map_pages(s_page, 1, 0);
976         }
977 }
978
979
980 #ifdef CONFIG_HIGHMEM
981 static inline struct page *
982 page_is_saveable(struct zone *zone, unsigned long pfn)
983 {
984         return is_highmem(zone) ?
985                 saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
986 }
987
988 static void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
989 {
990         struct page *s_page, *d_page;
991         void *src, *dst;
992
993         s_page = pfn_to_page(src_pfn);
994         d_page = pfn_to_page(dst_pfn);
995         if (PageHighMem(s_page)) {
996                 src = kmap_atomic(s_page, KM_USER0);
997                 dst = kmap_atomic(d_page, KM_USER1);
998                 do_copy_page(dst, src);
999                 kunmap_atomic(dst, KM_USER1);
1000                 kunmap_atomic(src, KM_USER0);
1001         } else {
1002                 if (PageHighMem(d_page)) {
1003                         /* Page pointed to by src may contain some kernel
1004                          * data modified by kmap_atomic()
1005                          */
1006                         safe_copy_page(buffer, s_page);
1007                         dst = kmap_atomic(d_page, KM_USER0);
1008                         copy_page(dst, buffer);
1009                         kunmap_atomic(dst, KM_USER0);
1010                 } else {
1011                         safe_copy_page(page_address(d_page), s_page);
1012                 }
1013         }
1014 }
1015 #else
1016 #define page_is_saveable(zone, pfn)     saveable_page(zone, pfn)
1017
1018 static inline void copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1019 {
1020         safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1021                                 pfn_to_page(src_pfn));
1022 }
1023 #endif /* CONFIG_HIGHMEM */
1024
1025 static void
1026 copy_data_pages(struct memory_bitmap *copy_bm, struct memory_bitmap *orig_bm)
1027 {
1028         struct zone *zone;
1029         unsigned long pfn;
1030
1031         for_each_populated_zone(zone) {
1032                 unsigned long max_zone_pfn;
1033
1034                 mark_free_pages(zone);
1035                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1036                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1037                         if (page_is_saveable(zone, pfn))
1038                                 memory_bm_set_bit(orig_bm, pfn);
1039         }
1040         memory_bm_position_reset(orig_bm);
1041         memory_bm_position_reset(copy_bm);
1042         for(;;) {
1043                 pfn = memory_bm_next_pfn(orig_bm);
1044                 if (unlikely(pfn == BM_END_OF_MAP))
1045                         break;
1046                 copy_data_page(memory_bm_next_pfn(copy_bm), pfn);
1047         }
1048 }
1049
1050 /* Total number of image pages */
1051 static unsigned int nr_copy_pages;
1052 /* Number of pages needed for saving the original pfns of the image pages */
1053 static unsigned int nr_meta_pages;
1054 /*
1055  * Numbers of normal and highmem page frames allocated for hibernation image
1056  * before suspending devices.
1057  */
1058 unsigned int alloc_normal, alloc_highmem;
1059 /*
1060  * Memory bitmap used for marking saveable pages (during hibernation) or
1061  * hibernation image pages (during restore)
1062  */
1063 static struct memory_bitmap orig_bm;
1064 /*
1065  * Memory bitmap used during hibernation for marking allocated page frames that
1066  * will contain copies of saveable pages.  During restore it is initially used
1067  * for marking hibernation image pages, but then the set bits from it are
1068  * duplicated in @orig_bm and it is released.  On highmem systems it is next
1069  * used for marking "safe" highmem pages, but it has to be reinitialized for
1070  * this purpose.
1071  */
1072 static struct memory_bitmap copy_bm;
1073
1074 /**
1075  *      swsusp_free - free pages allocated for the suspend.
1076  *
1077  *      Suspend pages are alocated before the atomic copy is made, so we
1078  *      need to release them after the resume.
1079  */
1080
1081 void swsusp_free(void)
1082 {
1083         struct zone *zone;
1084         unsigned long pfn, max_zone_pfn;
1085
1086         for_each_populated_zone(zone) {
1087                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1088                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1089                         if (pfn_valid(pfn)) {
1090                                 struct page *page = pfn_to_page(pfn);
1091
1092                                 if (swsusp_page_is_forbidden(page) &&
1093                                     swsusp_page_is_free(page)) {
1094                                         swsusp_unset_page_forbidden(page);
1095                                         swsusp_unset_page_free(page);
1096                                         __free_page(page);
1097                                 }
1098                         }
1099         }
1100         nr_copy_pages = 0;
1101         nr_meta_pages = 0;
1102         restore_pblist = NULL;
1103         buffer = NULL;
1104         alloc_normal = 0;
1105         alloc_highmem = 0;
1106 }
1107
1108 /* Helper functions used for the shrinking of memory. */
1109
1110 #define GFP_IMAGE       (GFP_KERNEL | __GFP_NOWARN)
1111
1112 /**
1113  * preallocate_image_pages - Allocate a number of pages for hibernation image
1114  * @nr_pages: Number of page frames to allocate.
1115  * @mask: GFP flags to use for the allocation.
1116  *
1117  * Return value: Number of page frames actually allocated
1118  */
1119 static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1120 {
1121         unsigned long nr_alloc = 0;
1122
1123         while (nr_pages > 0) {
1124                 struct page *page;
1125
1126                 page = alloc_image_page(mask);
1127                 if (!page)
1128                         break;
1129                 memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1130                 if (PageHighMem(page))
1131                         alloc_highmem++;
1132                 else
1133                         alloc_normal++;
1134                 nr_pages--;
1135                 nr_alloc++;
1136         }
1137
1138         return nr_alloc;
1139 }
1140
1141 static unsigned long preallocate_image_memory(unsigned long nr_pages,
1142                                               unsigned long avail_normal)
1143 {
1144         unsigned long alloc;
1145
1146         if (avail_normal <= alloc_normal)
1147                 return 0;
1148
1149         alloc = avail_normal - alloc_normal;
1150         if (nr_pages < alloc)
1151                 alloc = nr_pages;
1152
1153         return preallocate_image_pages(alloc, GFP_IMAGE);
1154 }
1155
1156 #ifdef CONFIG_HIGHMEM
1157 static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1158 {
1159         return preallocate_image_pages(nr_pages, GFP_IMAGE | __GFP_HIGHMEM);
1160 }
1161
1162 /**
1163  *  __fraction - Compute (an approximation of) x * (multiplier / base)
1164  */
1165 static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1166 {
1167         x *= multiplier;
1168         do_div(x, base);
1169         return (unsigned long)x;
1170 }
1171
1172 static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1173                                                 unsigned long highmem,
1174                                                 unsigned long total)
1175 {
1176         unsigned long alloc = __fraction(nr_pages, highmem, total);
1177
1178         return preallocate_image_pages(alloc, GFP_IMAGE | __GFP_HIGHMEM);
1179 }
1180 #else /* CONFIG_HIGHMEM */
1181 static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1182 {
1183         return 0;
1184 }
1185
1186 static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1187                                                 unsigned long highmem,
1188                                                 unsigned long total)
1189 {
1190         return 0;
1191 }
1192 #endif /* CONFIG_HIGHMEM */
1193
1194 /**
1195  * free_unnecessary_pages - Release preallocated pages not needed for the image
1196  */
1197 static void free_unnecessary_pages(void)
1198 {
1199         unsigned long save, to_free_normal, to_free_highmem;
1200
1201         save = count_data_pages();
1202         if (alloc_normal >= save) {
1203                 to_free_normal = alloc_normal - save;
1204                 save = 0;
1205         } else {
1206                 to_free_normal = 0;
1207                 save -= alloc_normal;
1208         }
1209         save += count_highmem_pages();
1210         if (alloc_highmem >= save) {
1211                 to_free_highmem = alloc_highmem - save;
1212         } else {
1213                 to_free_highmem = 0;
1214                 to_free_normal -= save - alloc_highmem;
1215         }
1216
1217         memory_bm_position_reset(&copy_bm);
1218
1219         while (to_free_normal > 0 || to_free_highmem > 0) {
1220                 unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1221                 struct page *page = pfn_to_page(pfn);
1222
1223                 if (PageHighMem(page)) {
1224                         if (!to_free_highmem)
1225                                 continue;
1226                         to_free_highmem--;
1227                         alloc_highmem--;
1228                 } else {
1229                         if (!to_free_normal)
1230                                 continue;
1231                         to_free_normal--;
1232                         alloc_normal--;
1233                 }
1234                 memory_bm_clear_bit(&copy_bm, pfn);
1235                 swsusp_unset_page_forbidden(page);
1236                 swsusp_unset_page_free(page);
1237                 __free_page(page);
1238         }
1239 }
1240
1241 /**
1242  * minimum_image_size - Estimate the minimum acceptable size of an image
1243  * @saveable: Number of saveable pages in the system.
1244  *
1245  * We want to avoid attempting to free too much memory too hard, so estimate the
1246  * minimum acceptable size of a hibernation image to use as the lower limit for
1247  * preallocating memory.
1248  *
1249  * We assume that the minimum image size should be proportional to
1250  *
1251  * [number of saveable pages] - [number of pages that can be freed in theory]
1252  *
1253  * where the second term is the sum of (1) reclaimable slab pages, (2) active
1254  * and (3) inactive anonymouns pages, (4) active and (5) inactive file pages,
1255  * minus mapped file pages.
1256  */
1257 static unsigned long minimum_image_size(unsigned long saveable)
1258 {
1259         unsigned long size;
1260
1261         size = global_page_state(NR_SLAB_RECLAIMABLE)
1262                 + global_page_state(NR_ACTIVE_ANON)
1263                 + global_page_state(NR_INACTIVE_ANON)
1264                 + global_page_state(NR_ACTIVE_FILE)
1265                 + global_page_state(NR_INACTIVE_FILE)
1266                 - global_page_state(NR_FILE_MAPPED);
1267
1268         return saveable <= size ? 0 : saveable - size;
1269 }
1270
1271 /**
1272  * hibernate_preallocate_memory - Preallocate memory for hibernation image
1273  *
1274  * To create a hibernation image it is necessary to make a copy of every page
1275  * frame in use.  We also need a number of page frames to be free during
1276  * hibernation for allocations made while saving the image and for device
1277  * drivers, in case they need to allocate memory from their hibernation
1278  * callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1279  * estimate) and reserverd_size divided by PAGE_SIZE (which is tunable through
1280  * /sys/power/reserved_size, respectively).  To make this happen, we compute the
1281  * total number of available page frames and allocate at least
1282  *
1283  * ([page frames total] + PAGES_FOR_IO + [metadata pages]) / 2
1284  *  + 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1285  *
1286  * of them, which corresponds to the maximum size of a hibernation image.
1287  *
1288  * If image_size is set below the number following from the above formula,
1289  * the preallocation of memory is continued until the total number of saveable
1290  * pages in the system is below the requested image size or the minimum
1291  * acceptable image size returned by minimum_image_size(), whichever is greater.
1292  */
1293 int hibernate_preallocate_memory(void)
1294 {
1295         struct zone *zone;
1296         unsigned long saveable, size, max_size, count, highmem, pages = 0;
1297         unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1298         struct timeval start, stop;
1299         int error;
1300
1301         printk(KERN_INFO "PM: Preallocating image memory... ");
1302         do_gettimeofday(&start);
1303
1304         error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1305         if (error)
1306                 goto err_out;
1307
1308         error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1309         if (error)
1310                 goto err_out;
1311
1312         alloc_normal = 0;
1313         alloc_highmem = 0;
1314
1315         /* Count the number of saveable data pages. */
1316         save_highmem = count_highmem_pages();
1317         saveable = count_data_pages();
1318
1319         /*
1320          * Compute the total number of page frames we can use (count) and the
1321          * number of pages needed for image metadata (size).
1322          */
1323         count = saveable;
1324         saveable += save_highmem;
1325         highmem = save_highmem;
1326         size = 0;
1327         for_each_populated_zone(zone) {
1328                 size += snapshot_additional_pages(zone);
1329                 if (is_highmem(zone))
1330                         highmem += zone_page_state(zone, NR_FREE_PAGES);
1331                 else
1332                         count += zone_page_state(zone, NR_FREE_PAGES);
1333         }
1334         avail_normal = count;
1335         count += highmem;
1336         count -= totalreserve_pages;
1337
1338         /* Compute the maximum number of saveable pages to leave in memory. */
1339         max_size = (count - (size + PAGES_FOR_IO)) / 2
1340                         - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1341         /* Compute the desired number of image pages specified by image_size. */
1342         size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1343         if (size > max_size)
1344                 size = max_size;
1345         /*
1346          * If the desired number of image pages is at least as large as the
1347          * current number of saveable pages in memory, allocate page frames for
1348          * the image and we're done.
1349          */
1350         if (size >= saveable) {
1351                 pages = preallocate_image_highmem(save_highmem);
1352                 pages += preallocate_image_memory(saveable - pages, avail_normal);
1353                 goto out;
1354         }
1355
1356         /* Estimate the minimum size of the image. */
1357         pages = minimum_image_size(saveable);
1358         /*
1359          * To avoid excessive pressure on the normal zone, leave room in it to
1360          * accommodate an image of the minimum size (unless it's already too
1361          * small, in which case don't preallocate pages from it at all).
1362          */
1363         if (avail_normal > pages)
1364                 avail_normal -= pages;
1365         else
1366                 avail_normal = 0;
1367         if (size < pages)
1368                 size = min_t(unsigned long, pages, max_size);
1369
1370         /*
1371          * Let the memory management subsystem know that we're going to need a
1372          * large number of page frames to allocate and make it free some memory.
1373          * NOTE: If this is not done, performance will be hurt badly in some
1374          * test cases.
1375          */
1376         shrink_all_memory(saveable - size);
1377
1378         /*
1379          * The number of saveable pages in memory was too high, so apply some
1380          * pressure to decrease it.  First, make room for the largest possible
1381          * image and fail if that doesn't work.  Next, try to decrease the size
1382          * of the image as much as indicated by 'size' using allocations from
1383          * highmem and non-highmem zones separately.
1384          */
1385         pages_highmem = preallocate_image_highmem(highmem / 2);
1386         alloc = (count - max_size) - pages_highmem;
1387         pages = preallocate_image_memory(alloc, avail_normal);
1388         if (pages < alloc) {
1389                 /* We have exhausted non-highmem pages, try highmem. */
1390                 alloc -= pages;
1391                 pages += pages_highmem;
1392                 pages_highmem = preallocate_image_highmem(alloc);
1393                 if (pages_highmem < alloc)
1394                         goto err_out;
1395                 pages += pages_highmem;
1396                 /*
1397                  * size is the desired number of saveable pages to leave in
1398                  * memory, so try to preallocate (all memory - size) pages.
1399                  */
1400                 alloc = (count - pages) - size;
1401                 pages += preallocate_image_highmem(alloc);
1402         } else {
1403                 /*
1404                  * There are approximately max_size saveable pages at this point
1405                  * and we want to reduce this number down to size.
1406                  */
1407                 alloc = max_size - size;
1408                 size = preallocate_highmem_fraction(alloc, highmem, count);
1409                 pages_highmem += size;
1410                 alloc -= size;
1411                 size = preallocate_image_memory(alloc, avail_normal);
1412                 pages_highmem += preallocate_image_highmem(alloc - size);
1413                 pages += pages_highmem + size;
1414         }
1415
1416         /*
1417          * We only need as many page frames for the image as there are saveable
1418          * pages in memory, but we have allocated more.  Release the excessive
1419          * ones now.
1420          */
1421         free_unnecessary_pages();
1422
1423  out:
1424         do_gettimeofday(&stop);
1425         printk(KERN_CONT "done (allocated %lu pages)\n", pages);
1426         swsusp_show_speed(&start, &stop, pages, "Allocated");
1427
1428         return 0;
1429
1430  err_out:
1431         printk(KERN_CONT "\n");
1432         swsusp_free();
1433         return -ENOMEM;
1434 }
1435
1436 #ifdef CONFIG_HIGHMEM
1437 /**
1438   *     count_pages_for_highmem - compute the number of non-highmem pages
1439   *     that will be necessary for creating copies of highmem pages.
1440   */
1441
1442 static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1443 {
1444         unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1445
1446         if (free_highmem >= nr_highmem)
1447                 nr_highmem = 0;
1448         else
1449                 nr_highmem -= free_highmem;
1450
1451         return nr_highmem;
1452 }
1453 #else
1454 static unsigned int
1455 count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
1456 #endif /* CONFIG_HIGHMEM */
1457
1458 /**
1459  *      enough_free_mem - Make sure we have enough free memory for the
1460  *      snapshot image.
1461  */
1462
1463 static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
1464 {
1465         struct zone *zone;
1466         unsigned int free = alloc_normal;
1467
1468         for_each_populated_zone(zone)
1469                 if (!is_highmem(zone))
1470                         free += zone_page_state(zone, NR_FREE_PAGES);
1471
1472         nr_pages += count_pages_for_highmem(nr_highmem);
1473         pr_debug("PM: Normal pages needed: %u + %u, available pages: %u\n",
1474                 nr_pages, PAGES_FOR_IO, free);
1475
1476         return free > nr_pages + PAGES_FOR_IO;
1477 }
1478
1479 #ifdef CONFIG_HIGHMEM
1480 /**
1481  *      get_highmem_buffer - if there are some highmem pages in the suspend
1482  *      image, we may need the buffer to copy them and/or load their data.
1483  */
1484
1485 static inline int get_highmem_buffer(int safe_needed)
1486 {
1487         buffer = get_image_page(GFP_ATOMIC | __GFP_COLD, safe_needed);
1488         return buffer ? 0 : -ENOMEM;
1489 }
1490
1491 /**
1492  *      alloc_highmem_image_pages - allocate some highmem pages for the image.
1493  *      Try to allocate as many pages as needed, but if the number of free
1494  *      highmem pages is lesser than that, allocate them all.
1495  */
1496
1497 static inline unsigned int
1498 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int nr_highmem)
1499 {
1500         unsigned int to_alloc = count_free_highmem_pages();
1501
1502         if (to_alloc > nr_highmem)
1503                 to_alloc = nr_highmem;
1504
1505         nr_highmem -= to_alloc;
1506         while (to_alloc-- > 0) {
1507                 struct page *page;
1508
1509                 page = alloc_image_page(__GFP_HIGHMEM);
1510                 memory_bm_set_bit(bm, page_to_pfn(page));
1511         }
1512         return nr_highmem;
1513 }
1514 #else
1515 static inline int get_highmem_buffer(int safe_needed) { return 0; }
1516
1517 static inline unsigned int
1518 alloc_highmem_pages(struct memory_bitmap *bm, unsigned int n) { return 0; }
1519 #endif /* CONFIG_HIGHMEM */
1520
1521 /**
1522  *      swsusp_alloc - allocate memory for the suspend image
1523  *
1524  *      We first try to allocate as many highmem pages as there are
1525  *      saveable highmem pages in the system.  If that fails, we allocate
1526  *      non-highmem pages for the copies of the remaining highmem ones.
1527  *
1528  *      In this approach it is likely that the copies of highmem pages will
1529  *      also be located in the high memory, because of the way in which
1530  *      copy_data_pages() works.
1531  */
1532
1533 static int
1534 swsusp_alloc(struct memory_bitmap *orig_bm, struct memory_bitmap *copy_bm,
1535                 unsigned int nr_pages, unsigned int nr_highmem)
1536 {
1537         if (nr_highmem > 0) {
1538                 if (get_highmem_buffer(PG_ANY))
1539                         goto err_out;
1540                 if (nr_highmem > alloc_highmem) {
1541                         nr_highmem -= alloc_highmem;
1542                         nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
1543                 }
1544         }
1545         if (nr_pages > alloc_normal) {
1546                 nr_pages -= alloc_normal;
1547                 while (nr_pages-- > 0) {
1548                         struct page *page;
1549
1550                         page = alloc_image_page(GFP_ATOMIC | __GFP_COLD);
1551                         if (!page)
1552                                 goto err_out;
1553                         memory_bm_set_bit(copy_bm, page_to_pfn(page));
1554                 }
1555         }
1556
1557         return 0;
1558
1559  err_out:
1560         swsusp_free();
1561         return -ENOMEM;
1562 }
1563
1564 asmlinkage int swsusp_save(void)
1565 {
1566         unsigned int nr_pages, nr_highmem;
1567
1568         printk(KERN_INFO "PM: Creating hibernation image:\n");
1569
1570         drain_local_pages(NULL);
1571         nr_pages = count_data_pages();
1572         nr_highmem = count_highmem_pages();
1573         printk(KERN_INFO "PM: Need to copy %u pages\n", nr_pages + nr_highmem);
1574
1575         if (!enough_free_mem(nr_pages, nr_highmem)) {
1576                 printk(KERN_ERR "PM: Not enough free memory\n");
1577                 return -ENOMEM;
1578         }
1579
1580         if (swsusp_alloc(&orig_bm, &copy_bm, nr_pages, nr_highmem)) {
1581                 printk(KERN_ERR "PM: Memory allocation failed\n");
1582                 return -ENOMEM;
1583         }
1584
1585         /* During allocating of suspend pagedir, new cold pages may appear.
1586          * Kill them.
1587          */
1588         drain_local_pages(NULL);
1589         copy_data_pages(&copy_bm, &orig_bm);
1590
1591         /*
1592          * End of critical section. From now on, we can write to memory,
1593          * but we should not touch disk. This specially means we must _not_
1594          * touch swap space! Except we must write out our image of course.
1595          */
1596
1597         nr_pages += nr_highmem;
1598         nr_copy_pages = nr_pages;
1599         nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
1600
1601         printk(KERN_INFO "PM: Hibernation image created (%d pages copied)\n",
1602                 nr_pages);
1603
1604         return 0;
1605 }
1606
1607 #ifndef CONFIG_ARCH_HIBERNATION_HEADER
1608 static int init_header_complete(struct swsusp_info *info)
1609 {
1610         memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
1611         info->version_code = LINUX_VERSION_CODE;
1612         return 0;
1613 }
1614
1615 static char *check_image_kernel(struct swsusp_info *info)
1616 {
1617         if (info->version_code != LINUX_VERSION_CODE)
1618                 return "kernel version";
1619         if (strcmp(info->uts.sysname,init_utsname()->sysname))
1620                 return "system type";
1621         if (strcmp(info->uts.release,init_utsname()->release))
1622                 return "kernel release";
1623         if (strcmp(info->uts.version,init_utsname()->version))
1624                 return "version";
1625         if (strcmp(info->uts.machine,init_utsname()->machine))
1626                 return "machine";
1627         return NULL;
1628 }
1629 #endif /* CONFIG_ARCH_HIBERNATION_HEADER */
1630
1631 unsigned long snapshot_get_image_size(void)
1632 {
1633         return nr_copy_pages + nr_meta_pages + 1;
1634 }
1635
1636 static int init_header(struct swsusp_info *info)
1637 {
1638         memset(info, 0, sizeof(struct swsusp_info));
1639         info->num_physpages = num_physpages;
1640         info->image_pages = nr_copy_pages;
1641         info->pages = snapshot_get_image_size();
1642         info->size = info->pages;
1643         info->size <<= PAGE_SHIFT;
1644         return init_header_complete(info);
1645 }
1646
1647 /**
1648  *      pack_pfns - pfns corresponding to the set bits found in the bitmap @bm
1649  *      are stored in the array @buf[] (1 page at a time)
1650  */
1651
1652 static inline void
1653 pack_pfns(unsigned long *buf, struct memory_bitmap *bm)
1654 {
1655         int j;
1656
1657         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1658                 buf[j] = memory_bm_next_pfn(bm);
1659                 if (unlikely(buf[j] == BM_END_OF_MAP))
1660                         break;
1661         }
1662 }
1663
1664 /**
1665  *      snapshot_read_next - used for reading the system memory snapshot.
1666  *
1667  *      On the first call to it @handle should point to a zeroed
1668  *      snapshot_handle structure.  The structure gets updated and a pointer
1669  *      to it should be passed to this function every next time.
1670  *
1671  *      On success the function returns a positive number.  Then, the caller
1672  *      is allowed to read up to the returned number of bytes from the memory
1673  *      location computed by the data_of() macro.
1674  *
1675  *      The function returns 0 to indicate the end of data stream condition,
1676  *      and a negative number is returned on error.  In such cases the
1677  *      structure pointed to by @handle is not updated and should not be used
1678  *      any more.
1679  */
1680
1681 int snapshot_read_next(struct snapshot_handle *handle)
1682 {
1683         if (handle->cur > nr_meta_pages + nr_copy_pages)
1684                 return 0;
1685
1686         if (!buffer) {
1687                 /* This makes the buffer be freed by swsusp_free() */
1688                 buffer = get_image_page(GFP_ATOMIC, PG_ANY);
1689                 if (!buffer)
1690                         return -ENOMEM;
1691         }
1692         if (!handle->cur) {
1693                 int error;
1694
1695                 error = init_header((struct swsusp_info *)buffer);
1696                 if (error)
1697                         return error;
1698                 handle->buffer = buffer;
1699                 memory_bm_position_reset(&orig_bm);
1700                 memory_bm_position_reset(&copy_bm);
1701         } else if (handle->cur <= nr_meta_pages) {
1702                 clear_page(buffer);
1703                 pack_pfns(buffer, &orig_bm);
1704         } else {
1705                 struct page *page;
1706
1707                 page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
1708                 if (PageHighMem(page)) {
1709                         /* Highmem pages are copied to the buffer,
1710                          * because we can't return with a kmapped
1711                          * highmem page (we may not be called again).
1712                          */
1713                         void *kaddr;
1714
1715                         kaddr = kmap_atomic(page, KM_USER0);
1716                         copy_page(buffer, kaddr);
1717                         kunmap_atomic(kaddr, KM_USER0);
1718                         handle->buffer = buffer;
1719                 } else {
1720                         handle->buffer = page_address(page);
1721                 }
1722         }
1723         handle->cur++;
1724         return PAGE_SIZE;
1725 }
1726
1727 /**
1728  *      mark_unsafe_pages - mark the pages that cannot be used for storing
1729  *      the image during resume, because they conflict with the pages that
1730  *      had been used before suspend
1731  */
1732
1733 static int mark_unsafe_pages(struct memory_bitmap *bm)
1734 {
1735         struct zone *zone;
1736         unsigned long pfn, max_zone_pfn;
1737
1738         /* Clear page flags */
1739         for_each_populated_zone(zone) {
1740                 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1741                 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1742                         if (pfn_valid(pfn))
1743                                 swsusp_unset_page_free(pfn_to_page(pfn));
1744         }
1745
1746         /* Mark pages that correspond to the "original" pfns as "unsafe" */
1747         memory_bm_position_reset(bm);
1748         do {
1749                 pfn = memory_bm_next_pfn(bm);
1750                 if (likely(pfn != BM_END_OF_MAP)) {
1751                         if (likely(pfn_valid(pfn)))
1752                                 swsusp_set_page_free(pfn_to_page(pfn));
1753                         else
1754                                 return -EFAULT;
1755                 }
1756         } while (pfn != BM_END_OF_MAP);
1757
1758         allocated_unsafe_pages = 0;
1759
1760         return 0;
1761 }
1762
1763 static void
1764 duplicate_memory_bitmap(struct memory_bitmap *dst, struct memory_bitmap *src)
1765 {
1766         unsigned long pfn;
1767
1768         memory_bm_position_reset(src);
1769         pfn = memory_bm_next_pfn(src);
1770         while (pfn != BM_END_OF_MAP) {
1771                 memory_bm_set_bit(dst, pfn);
1772                 pfn = memory_bm_next_pfn(src);
1773         }
1774 }
1775
1776 static int check_header(struct swsusp_info *info)
1777 {
1778         char *reason;
1779
1780         reason = check_image_kernel(info);
1781         if (!reason && info->num_physpages != num_physpages)
1782                 reason = "memory size";
1783         if (reason) {
1784                 printk(KERN_ERR "PM: Image mismatch: %s\n", reason);
1785                 return -EPERM;
1786         }
1787         return 0;
1788 }
1789
1790 /**
1791  *      load header - check the image header and copy data from it
1792  */
1793
1794 static int
1795 load_header(struct swsusp_info *info)
1796 {
1797         int error;
1798
1799         restore_pblist = NULL;
1800         error = check_header(info);
1801         if (!error) {
1802                 nr_copy_pages = info->image_pages;
1803                 nr_meta_pages = info->pages - info->image_pages - 1;
1804         }
1805         return error;
1806 }
1807
1808 /**
1809  *      unpack_orig_pfns - for each element of @buf[] (1 page at a time) set
1810  *      the corresponding bit in the memory bitmap @bm
1811  */
1812 static int unpack_orig_pfns(unsigned long *buf, struct memory_bitmap *bm)
1813 {
1814         int j;
1815
1816         for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
1817                 if (unlikely(buf[j] == BM_END_OF_MAP))
1818                         break;
1819
1820                 if (memory_bm_pfn_present(bm, buf[j]))
1821                         memory_bm_set_bit(bm, buf[j]);
1822                 else
1823                         return -EFAULT;
1824         }
1825
1826         return 0;
1827 }
1828
1829 /* List of "safe" pages that may be used to store data loaded from the suspend
1830  * image
1831  */
1832 static struct linked_page *safe_pages_list;
1833
1834 #ifdef CONFIG_HIGHMEM
1835 /* struct highmem_pbe is used for creating the list of highmem pages that
1836  * should be restored atomically during the resume from disk, because the page
1837  * frames they have occupied before the suspend are in use.
1838  */
1839 struct highmem_pbe {
1840         struct page *copy_page; /* data is here now */
1841         struct page *orig_page; /* data was here before the suspend */
1842         struct highmem_pbe *next;
1843 };
1844
1845 /* List of highmem PBEs needed for restoring the highmem pages that were
1846  * allocated before the suspend and included in the suspend image, but have
1847  * also been allocated by the "resume" kernel, so their contents cannot be
1848  * written directly to their "original" page frames.
1849  */
1850 static struct highmem_pbe *highmem_pblist;
1851
1852 /**
1853  *      count_highmem_image_pages - compute the number of highmem pages in the
1854  *      suspend image.  The bits in the memory bitmap @bm that correspond to the
1855  *      image pages are assumed to be set.
1856  */
1857
1858 static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
1859 {
1860         unsigned long pfn;
1861         unsigned int cnt = 0;
1862
1863         memory_bm_position_reset(bm);
1864         pfn = memory_bm_next_pfn(bm);
1865         while (pfn != BM_END_OF_MAP) {
1866                 if (PageHighMem(pfn_to_page(pfn)))
1867                         cnt++;
1868
1869                 pfn = memory_bm_next_pfn(bm);
1870         }
1871         return cnt;
1872 }
1873
1874 /**
1875  *      prepare_highmem_image - try to allocate as many highmem pages as
1876  *      there are highmem image pages (@nr_highmem_p points to the variable
1877  *      containing the number of highmem image pages).  The pages that are
1878  *      "safe" (ie. will not be overwritten when the suspend image is
1879  *      restored) have the corresponding bits set in @bm (it must be
1880  *      unitialized).
1881  *
1882  *      NOTE: This function should not be called if there are no highmem
1883  *      image pages.
1884  */
1885
1886 static unsigned int safe_highmem_pages;
1887
1888 static struct memory_bitmap *safe_highmem_bm;
1889
1890 static int
1891 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
1892 {
1893         unsigned int to_alloc;
1894
1895         if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
1896                 return -ENOMEM;
1897
1898         if (get_highmem_buffer(PG_SAFE))
1899                 return -ENOMEM;
1900
1901         to_alloc = count_free_highmem_pages();
1902         if (to_alloc > *nr_highmem_p)
1903                 to_alloc = *nr_highmem_p;
1904         else
1905                 *nr_highmem_p = to_alloc;
1906
1907         safe_highmem_pages = 0;
1908         while (to_alloc-- > 0) {
1909                 struct page *page;
1910
1911                 page = alloc_page(__GFP_HIGHMEM);
1912                 if (!swsusp_page_is_free(page)) {
1913                         /* The page is "safe", set its bit the bitmap */
1914                         memory_bm_set_bit(bm, page_to_pfn(page));
1915                         safe_highmem_pages++;
1916                 }
1917                 /* Mark the page as allocated */
1918                 swsusp_set_page_forbidden(page);
1919                 swsusp_set_page_free(page);
1920         }
1921         memory_bm_position_reset(bm);
1922         safe_highmem_bm = bm;
1923         return 0;
1924 }
1925
1926 /**
1927  *      get_highmem_page_buffer - for given highmem image page find the buffer
1928  *      that suspend_write_next() should set for its caller to write to.
1929  *
1930  *      If the page is to be saved to its "original" page frame or a copy of
1931  *      the page is to be made in the highmem, @buffer is returned.  Otherwise,
1932  *      the copy of the page is to be made in normal memory, so the address of
1933  *      the copy is returned.
1934  *
1935  *      If @buffer is returned, the caller of suspend_write_next() will write
1936  *      the page's contents to @buffer, so they will have to be copied to the
1937  *      right location on the next call to suspend_write_next() and it is done
1938  *      with the help of copy_last_highmem_page().  For this purpose, if
1939  *      @buffer is returned, @last_highmem page is set to the page to which
1940  *      the data will have to be copied from @buffer.
1941  */
1942
1943 static struct page *last_highmem_page;
1944
1945 static void *
1946 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
1947 {
1948         struct highmem_pbe *pbe;
1949         void *kaddr;
1950
1951         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
1952                 /* We have allocated the "original" page frame and we can
1953                  * use it directly to store the loaded page.
1954                  */
1955                 last_highmem_page = page;
1956                 return buffer;
1957         }
1958         /* The "original" page frame has not been allocated and we have to
1959          * use a "safe" page frame to store the loaded page.
1960          */
1961         pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
1962         if (!pbe) {
1963                 swsusp_free();
1964                 return ERR_PTR(-ENOMEM);
1965         }
1966         pbe->orig_page = page;
1967         if (safe_highmem_pages > 0) {
1968                 struct page *tmp;
1969
1970                 /* Copy of the page will be stored in high memory */
1971                 kaddr = buffer;
1972                 tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
1973                 safe_highmem_pages--;
1974                 last_highmem_page = tmp;
1975                 pbe->copy_page = tmp;
1976         } else {
1977                 /* Copy of the page will be stored in normal memory */
1978                 kaddr = safe_pages_list;
1979                 safe_pages_list = safe_pages_list->next;
1980                 pbe->copy_page = virt_to_page(kaddr);
1981         }
1982         pbe->next = highmem_pblist;
1983         highmem_pblist = pbe;
1984         return kaddr;
1985 }
1986
1987 /**
1988  *      copy_last_highmem_page - copy the contents of a highmem image from
1989  *      @buffer, where the caller of snapshot_write_next() has place them,
1990  *      to the right location represented by @last_highmem_page .
1991  */
1992
1993 static void copy_last_highmem_page(void)
1994 {
1995         if (last_highmem_page) {
1996                 void *dst;
1997
1998                 dst = kmap_atomic(last_highmem_page, KM_USER0);
1999                 copy_page(dst, buffer);
2000                 kunmap_atomic(dst, KM_USER0);
2001                 last_highmem_page = NULL;
2002         }
2003 }
2004
2005 static inline int last_highmem_page_copied(void)
2006 {
2007         return !last_highmem_page;
2008 }
2009
2010 static inline void free_highmem_data(void)
2011 {
2012         if (safe_highmem_bm)
2013                 memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2014
2015         if (buffer)
2016                 free_image_page(buffer, PG_UNSAFE_CLEAR);
2017 }
2018 #else
2019 static inline int get_safe_write_buffer(void) { return 0; }
2020
2021 static unsigned int
2022 count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2023
2024 static inline int
2025 prepare_highmem_image(struct memory_bitmap *bm, unsigned int *nr_highmem_p)
2026 {
2027         return 0;
2028 }
2029
2030 static inline void *
2031 get_highmem_page_buffer(struct page *page, struct chain_allocator *ca)
2032 {
2033         return ERR_PTR(-EINVAL);
2034 }
2035
2036 static inline void copy_last_highmem_page(void) {}
2037 static inline int last_highmem_page_copied(void) { return 1; }
2038 static inline void free_highmem_data(void) {}
2039 #endif /* CONFIG_HIGHMEM */
2040
2041 /**
2042  *      prepare_image - use the memory bitmap @bm to mark the pages that will
2043  *      be overwritten in the process of restoring the system memory state
2044  *      from the suspend image ("unsafe" pages) and allocate memory for the
2045  *      image.
2046  *
2047  *      The idea is to allocate a new memory bitmap first and then allocate
2048  *      as many pages as needed for the image data, but not to assign these
2049  *      pages to specific tasks initially.  Instead, we just mark them as
2050  *      allocated and create a lists of "safe" pages that will be used
2051  *      later.  On systems with high memory a list of "safe" highmem pages is
2052  *      also created.
2053  */
2054
2055 #define PBES_PER_LINKED_PAGE    (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2056
2057 static int
2058 prepare_image(struct memory_bitmap *new_bm, struct memory_bitmap *bm)
2059 {
2060         unsigned int nr_pages, nr_highmem;
2061         struct linked_page *sp_list, *lp;
2062         int error;
2063
2064         /* If there is no highmem, the buffer will not be necessary */
2065         free_image_page(buffer, PG_UNSAFE_CLEAR);
2066         buffer = NULL;
2067
2068         nr_highmem = count_highmem_image_pages(bm);
2069         error = mark_unsafe_pages(bm);
2070         if (error)
2071                 goto Free;
2072
2073         error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2074         if (error)
2075                 goto Free;
2076
2077         duplicate_memory_bitmap(new_bm, bm);
2078         memory_bm_free(bm, PG_UNSAFE_KEEP);
2079         if (nr_highmem > 0) {
2080                 error = prepare_highmem_image(bm, &nr_highmem);
2081                 if (error)
2082                         goto Free;
2083         }
2084         /* Reserve some safe pages for potential later use.
2085          *
2086          * NOTE: This way we make sure there will be enough safe pages for the
2087          * chain_alloc() in get_buffer().  It is a bit wasteful, but
2088          * nr_copy_pages cannot be greater than 50% of the memory anyway.
2089          */
2090         sp_list = NULL;
2091         /* nr_copy_pages cannot be lesser than allocated_unsafe_pages */
2092         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2093         nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2094         while (nr_pages > 0) {
2095                 lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2096                 if (!lp) {
2097                         error = -ENOMEM;
2098                         goto Free;
2099                 }
2100                 lp->next = sp_list;
2101                 sp_list = lp;
2102                 nr_pages--;
2103         }
2104         /* Preallocate memory for the image */
2105         safe_pages_list = NULL;
2106         nr_pages = nr_copy_pages - nr_highmem - allocated_unsafe_pages;
2107         while (nr_pages > 0) {
2108                 lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2109                 if (!lp) {
2110                         error = -ENOMEM;
2111                         goto Free;
2112                 }
2113                 if (!swsusp_page_is_free(virt_to_page(lp))) {
2114                         /* The page is "safe", add it to the list */
2115                         lp->next = safe_pages_list;
2116                         safe_pages_list = lp;
2117                 }
2118                 /* Mark the page as allocated */
2119                 swsusp_set_page_forbidden(virt_to_page(lp));
2120                 swsusp_set_page_free(virt_to_page(lp));
2121                 nr_pages--;
2122         }
2123         /* Free the reserved safe pages so that chain_alloc() can use them */
2124         while (sp_list) {
2125                 lp = sp_list->next;
2126                 free_image_page(sp_list, PG_UNSAFE_CLEAR);
2127                 sp_list = lp;
2128         }
2129         return 0;
2130
2131  Free:
2132         swsusp_free();
2133         return error;
2134 }
2135
2136 /**
2137  *      get_buffer - compute the address that snapshot_write_next() should
2138  *      set for its caller to write to.
2139  */
2140
2141 static void *get_buffer(struct memory_bitmap *bm, struct chain_allocator *ca)
2142 {
2143         struct pbe *pbe;
2144         struct page *page;
2145         unsigned long pfn = memory_bm_next_pfn(bm);
2146
2147         if (pfn == BM_END_OF_MAP)
2148                 return ERR_PTR(-EFAULT);
2149
2150         page = pfn_to_page(pfn);
2151         if (PageHighMem(page))
2152                 return get_highmem_page_buffer(page, ca);
2153
2154         if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2155                 /* We have allocated the "original" page frame and we can
2156                  * use it directly to store the loaded page.
2157                  */
2158                 return page_address(page);
2159
2160         /* The "original" page frame has not been allocated and we have to
2161          * use a "safe" page frame to store the loaded page.
2162          */
2163         pbe = chain_alloc(ca, sizeof(struct pbe));
2164         if (!pbe) {
2165                 swsusp_free();
2166                 return ERR_PTR(-ENOMEM);
2167         }
2168         pbe->orig_address = page_address(page);
2169         pbe->address = safe_pages_list;
2170         safe_pages_list = safe_pages_list->next;
2171         pbe->next = restore_pblist;
2172         restore_pblist = pbe;
2173         return pbe->address;
2174 }
2175
2176 /**
2177  *      snapshot_write_next - used for writing the system memory snapshot.
2178  *
2179  *      On the first call to it @handle should point to a zeroed
2180  *      snapshot_handle structure.  The structure gets updated and a pointer
2181  *      to it should be passed to this function every next time.
2182  *
2183  *      On success the function returns a positive number.  Then, the caller
2184  *      is allowed to write up to the returned number of bytes to the memory
2185  *      location computed by the data_of() macro.
2186  *
2187  *      The function returns 0 to indicate the "end of file" condition,
2188  *      and a negative number is returned on error.  In such cases the
2189  *      structure pointed to by @handle is not updated and should not be used
2190  *      any more.
2191  */
2192
2193 int snapshot_write_next(struct snapshot_handle *handle)
2194 {
2195         static struct chain_allocator ca;
2196         int error = 0;
2197
2198         /* Check if we have already loaded the entire image */
2199         if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages)
2200                 return 0;
2201
2202         handle->sync_read = 1;
2203
2204         if (!handle->cur) {
2205                 if (!buffer)
2206                         /* This makes the buffer be freed by swsusp_free() */
2207                         buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2208
2209                 if (!buffer)
2210                         return -ENOMEM;
2211
2212                 handle->buffer = buffer;
2213         } else if (handle->cur == 1) {
2214                 error = load_header(buffer);
2215                 if (error)
2216                         return error;
2217
2218                 error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2219                 if (error)
2220                         return error;
2221
2222         } else if (handle->cur <= nr_meta_pages + 1) {
2223                 error = unpack_orig_pfns(buffer, &copy_bm);
2224                 if (error)
2225                         return error;
2226
2227                 if (handle->cur == nr_meta_pages + 1) {
2228                         error = prepare_image(&orig_bm, &copy_bm);
2229                         if (error)
2230                                 return error;
2231
2232                         chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2233                         memory_bm_position_reset(&orig_bm);
2234                         restore_pblist = NULL;
2235                         handle->buffer = get_buffer(&orig_bm, &ca);
2236                         handle->sync_read = 0;
2237                         if (IS_ERR(handle->buffer))
2238                                 return PTR_ERR(handle->buffer);
2239                 }
2240         } else {
2241                 copy_last_highmem_page();
2242                 handle->buffer = get_buffer(&orig_bm, &ca);
2243                 if (IS_ERR(handle->buffer))
2244                         return PTR_ERR(handle->buffer);
2245                 if (handle->buffer != buffer)
2246                         handle->sync_read = 0;
2247         }
2248         handle->cur++;
2249         return PAGE_SIZE;
2250 }
2251
2252 /**
2253  *      snapshot_write_finalize - must be called after the last call to
2254  *      snapshot_write_next() in case the last page in the image happens
2255  *      to be a highmem page and its contents should be stored in the
2256  *      highmem.  Additionally, it releases the memory that will not be
2257  *      used any more.
2258  */
2259
2260 void snapshot_write_finalize(struct snapshot_handle *handle)
2261 {
2262         copy_last_highmem_page();
2263         /* Free only if we have loaded the image entirely */
2264         if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages) {
2265                 memory_bm_free(&orig_bm, PG_UNSAFE_CLEAR);
2266                 free_highmem_data();
2267         }
2268 }
2269
2270 int snapshot_image_loaded(struct snapshot_handle *handle)
2271 {
2272         return !(!nr_copy_pages || !last_highmem_page_copied() ||
2273                         handle->cur <= nr_meta_pages + nr_copy_pages);
2274 }
2275
2276 #ifdef CONFIG_HIGHMEM
2277 /* Assumes that @buf is ready and points to a "safe" page */
2278 static inline void
2279 swap_two_pages_data(struct page *p1, struct page *p2, void *buf)
2280 {
2281         void *kaddr1, *kaddr2;
2282
2283         kaddr1 = kmap_atomic(p1, KM_USER0);
2284         kaddr2 = kmap_atomic(p2, KM_USER1);
2285         copy_page(buf, kaddr1);
2286         copy_page(kaddr1, kaddr2);
2287         copy_page(kaddr2, buf);
2288         kunmap_atomic(kaddr2, KM_USER1);
2289         kunmap_atomic(kaddr1, KM_USER0);
2290 }
2291
2292 /**
2293  *      restore_highmem - for each highmem page that was allocated before
2294  *      the suspend and included in the suspend image, and also has been
2295  *      allocated by the "resume" kernel swap its current (ie. "before
2296  *      resume") contents with the previous (ie. "before suspend") one.
2297  *
2298  *      If the resume eventually fails, we can call this function once
2299  *      again and restore the "before resume" highmem state.
2300  */
2301
2302 int restore_highmem(void)
2303 {
2304         struct highmem_pbe *pbe = highmem_pblist;
2305         void *buf;
2306
2307         if (!pbe)
2308                 return 0;
2309
2310         buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2311         if (!buf)
2312                 return -ENOMEM;
2313
2314         while (pbe) {
2315                 swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2316                 pbe = pbe->next;
2317         }
2318         free_image_page(buf, PG_UNSAFE_CLEAR);
2319         return 0;
2320 }
2321 #endif /* CONFIG_HIGHMEM */