eef311a58a649821f60532602a085f2a79b014bc
[linux-2.6.git] / kernel / power / swap.c
1 /*
2  * linux/kernel/power/swap.c
3  *
4  * This file provides functions for reading the suspend image from
5  * and writing it to a swap partition.
6  *
7  * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
8  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
9  * Copyright (C) 2010 Bojan Smojver <bojan@rexursive.com>
10  *
11  * This file is released under the GPLv2.
12  *
13  */
14
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/genhd.h>
20 #include <linux/device.h>
21 #include <linux/bio.h>
22 #include <linux/blkdev.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/pm.h>
26 #include <linux/slab.h>
27 #include <linux/lzo.h>
28 #include <linux/vmalloc.h>
29 #include <linux/cpumask.h>
30 #include <linux/atomic.h>
31 #include <linux/kthread.h>
32 #include <linux/crc32.h>
33
34 #include "power.h"
35
36 #define HIBERNATE_SIG   "S1SUSPEND"
37
38 /*
39  *      The swap map is a data structure used for keeping track of each page
40  *      written to a swap partition.  It consists of many swap_map_page
41  *      structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
42  *      These structures are stored on the swap and linked together with the
43  *      help of the .next_swap member.
44  *
45  *      The swap map is created during suspend.  The swap map pages are
46  *      allocated and populated one at a time, so we only need one memory
47  *      page to set up the entire structure.
48  *
49  *      During resume we pick up all swap_map_page structures into a list.
50  */
51
52 #define MAP_PAGE_ENTRIES        (PAGE_SIZE / sizeof(sector_t) - 1)
53
54 /*
55  * Number of free pages that are not high.
56  */
57 static inline unsigned long low_free_pages(void)
58 {
59         return nr_free_pages() - nr_free_highpages();
60 }
61
62 /*
63  * Number of pages required to be kept free while writing the image. Always
64  * half of all available low pages before the writing starts.
65  */
66 static inline unsigned long reqd_free_pages(void)
67 {
68         return low_free_pages() / 2;
69 }
70
71 struct swap_map_page {
72         sector_t entries[MAP_PAGE_ENTRIES];
73         sector_t next_swap;
74 };
75
76 struct swap_map_page_list {
77         struct swap_map_page *map;
78         struct swap_map_page_list *next;
79 };
80
81 /**
82  *      The swap_map_handle structure is used for handling swap in
83  *      a file-alike way
84  */
85
86 struct swap_map_handle {
87         struct swap_map_page *cur;
88         struct swap_map_page_list *maps;
89         sector_t cur_swap;
90         sector_t first_sector;
91         unsigned int k;
92         unsigned long reqd_free_pages;
93         u32 crc32;
94 };
95
96 struct swsusp_header {
97         char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
98                       sizeof(u32)];
99         u32     crc32;
100         sector_t image;
101         unsigned int flags;     /* Flags to pass to the "boot" kernel */
102         char    orig_sig[10];
103         char    sig[10];
104 } __attribute__((packed));
105
106 static struct swsusp_header *swsusp_header;
107
108 /**
109  *      The following functions are used for tracing the allocated
110  *      swap pages, so that they can be freed in case of an error.
111  */
112
113 struct swsusp_extent {
114         struct rb_node node;
115         unsigned long start;
116         unsigned long end;
117 };
118
119 static struct rb_root swsusp_extents = RB_ROOT;
120
121 static int swsusp_extents_insert(unsigned long swap_offset)
122 {
123         struct rb_node **new = &(swsusp_extents.rb_node);
124         struct rb_node *parent = NULL;
125         struct swsusp_extent *ext;
126
127         /* Figure out where to put the new node */
128         while (*new) {
129                 ext = container_of(*new, struct swsusp_extent, node);
130                 parent = *new;
131                 if (swap_offset < ext->start) {
132                         /* Try to merge */
133                         if (swap_offset == ext->start - 1) {
134                                 ext->start--;
135                                 return 0;
136                         }
137                         new = &((*new)->rb_left);
138                 } else if (swap_offset > ext->end) {
139                         /* Try to merge */
140                         if (swap_offset == ext->end + 1) {
141                                 ext->end++;
142                                 return 0;
143                         }
144                         new = &((*new)->rb_right);
145                 } else {
146                         /* It already is in the tree */
147                         return -EINVAL;
148                 }
149         }
150         /* Add the new node and rebalance the tree. */
151         ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
152         if (!ext)
153                 return -ENOMEM;
154
155         ext->start = swap_offset;
156         ext->end = swap_offset;
157         rb_link_node(&ext->node, parent, new);
158         rb_insert_color(&ext->node, &swsusp_extents);
159         return 0;
160 }
161
162 /**
163  *      alloc_swapdev_block - allocate a swap page and register that it has
164  *      been allocated, so that it can be freed in case of an error.
165  */
166
167 sector_t alloc_swapdev_block(int swap)
168 {
169         unsigned long offset;
170
171         offset = swp_offset(get_swap_page_of_type(swap));
172         if (offset) {
173                 if (swsusp_extents_insert(offset))
174                         swap_free(swp_entry(swap, offset));
175                 else
176                         return swapdev_block(swap, offset);
177         }
178         return 0;
179 }
180
181 /**
182  *      free_all_swap_pages - free swap pages allocated for saving image data.
183  *      It also frees the extents used to register which swap entries had been
184  *      allocated.
185  */
186
187 void free_all_swap_pages(int swap)
188 {
189         struct rb_node *node;
190
191         while ((node = swsusp_extents.rb_node)) {
192                 struct swsusp_extent *ext;
193                 unsigned long offset;
194
195                 ext = container_of(node, struct swsusp_extent, node);
196                 rb_erase(node, &swsusp_extents);
197                 for (offset = ext->start; offset <= ext->end; offset++)
198                         swap_free(swp_entry(swap, offset));
199
200                 kfree(ext);
201         }
202 }
203
204 int swsusp_swap_in_use(void)
205 {
206         return (swsusp_extents.rb_node != NULL);
207 }
208
209 /*
210  * General things
211  */
212
213 static unsigned short root_swap = 0xffff;
214 struct block_device *hib_resume_bdev;
215
216 /*
217  * Saving part
218  */
219
220 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
221 {
222         int error;
223
224         hib_bio_read_page(swsusp_resume_block, swsusp_header, NULL);
225         if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
226             !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
227                 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
228                 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
229                 swsusp_header->image = handle->first_sector;
230                 swsusp_header->flags = flags;
231                 if (flags & SF_CRC32_MODE)
232                         swsusp_header->crc32 = handle->crc32;
233                 error = hib_bio_write_page(swsusp_resume_block,
234                                         swsusp_header, NULL);
235         } else {
236                 printk(KERN_ERR "PM: Swap header not found!\n");
237                 error = -ENODEV;
238         }
239         return error;
240 }
241
242 /**
243  *      swsusp_swap_check - check if the resume device is a swap device
244  *      and get its index (if so)
245  *
246  *      This is called before saving image
247  */
248 static int swsusp_swap_check(void)
249 {
250         int res;
251
252         res = swap_type_of(swsusp_resume_device, swsusp_resume_block,
253                         &hib_resume_bdev);
254         if (res < 0)
255                 return res;
256
257         root_swap = res;
258         res = blkdev_get(hib_resume_bdev, FMODE_WRITE, NULL);
259         if (res)
260                 return res;
261
262         res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
263         if (res < 0)
264                 blkdev_put(hib_resume_bdev, FMODE_WRITE);
265
266         return res;
267 }
268
269 /**
270  *      write_page - Write one page to given swap location.
271  *      @buf:           Address we're writing.
272  *      @offset:        Offset of the swap page we're writing to.
273  *      @bio_chain:     Link the next write BIO here
274  */
275
276 static int write_page(void *buf, sector_t offset, struct bio **bio_chain)
277 {
278         void *src;
279         int ret;
280
281         if (!offset)
282                 return -ENOSPC;
283
284         if (bio_chain) {
285                 src = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
286                 if (src) {
287                         copy_page(src, buf);
288                 } else {
289                         ret = hib_wait_on_bio_chain(bio_chain); /* Free pages */
290                         if (ret)
291                                 return ret;
292                         src = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
293                         if (src) {
294                                 copy_page(src, buf);
295                         } else {
296                                 WARN_ON_ONCE(1);
297                                 bio_chain = NULL;       /* Go synchronous */
298                                 src = buf;
299                         }
300                 }
301         } else {
302                 src = buf;
303         }
304         return hib_bio_write_page(offset, src, bio_chain);
305 }
306
307 static void release_swap_writer(struct swap_map_handle *handle)
308 {
309         if (handle->cur)
310                 free_page((unsigned long)handle->cur);
311         handle->cur = NULL;
312 }
313
314 static int get_swap_writer(struct swap_map_handle *handle)
315 {
316         int ret;
317
318         ret = swsusp_swap_check();
319         if (ret) {
320                 if (ret != -ENOSPC)
321                         printk(KERN_ERR "PM: Cannot find swap device, try "
322                                         "swapon -a.\n");
323                 return ret;
324         }
325         handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
326         if (!handle->cur) {
327                 ret = -ENOMEM;
328                 goto err_close;
329         }
330         handle->cur_swap = alloc_swapdev_block(root_swap);
331         if (!handle->cur_swap) {
332                 ret = -ENOSPC;
333                 goto err_rel;
334         }
335         handle->k = 0;
336         handle->reqd_free_pages = reqd_free_pages();
337         handle->first_sector = handle->cur_swap;
338         return 0;
339 err_rel:
340         release_swap_writer(handle);
341 err_close:
342         swsusp_close(FMODE_WRITE);
343         return ret;
344 }
345
346 static int swap_write_page(struct swap_map_handle *handle, void *buf,
347                                 struct bio **bio_chain)
348 {
349         int error = 0;
350         sector_t offset;
351
352         if (!handle->cur)
353                 return -EINVAL;
354         offset = alloc_swapdev_block(root_swap);
355         error = write_page(buf, offset, bio_chain);
356         if (error)
357                 return error;
358         handle->cur->entries[handle->k++] = offset;
359         if (handle->k >= MAP_PAGE_ENTRIES) {
360                 offset = alloc_swapdev_block(root_swap);
361                 if (!offset)
362                         return -ENOSPC;
363                 handle->cur->next_swap = offset;
364                 error = write_page(handle->cur, handle->cur_swap, bio_chain);
365                 if (error)
366                         goto out;
367                 clear_page(handle->cur);
368                 handle->cur_swap = offset;
369                 handle->k = 0;
370         }
371         if (bio_chain && low_free_pages() <= handle->reqd_free_pages) {
372                 error = hib_wait_on_bio_chain(bio_chain);
373                 if (error)
374                         goto out;
375                 handle->reqd_free_pages = reqd_free_pages();
376         }
377  out:
378         return error;
379 }
380
381 static int flush_swap_writer(struct swap_map_handle *handle)
382 {
383         if (handle->cur && handle->cur_swap)
384                 return write_page(handle->cur, handle->cur_swap, NULL);
385         else
386                 return -EINVAL;
387 }
388
389 static int swap_writer_finish(struct swap_map_handle *handle,
390                 unsigned int flags, int error)
391 {
392         if (!error) {
393                 flush_swap_writer(handle);
394                 printk(KERN_INFO "PM: S");
395                 error = mark_swapfiles(handle, flags);
396                 printk("|\n");
397         }
398
399         if (error)
400                 free_all_swap_pages(root_swap);
401         release_swap_writer(handle);
402         swsusp_close(FMODE_WRITE);
403
404         return error;
405 }
406
407 /* We need to remember how much compressed data we need to read. */
408 #define LZO_HEADER      sizeof(size_t)
409
410 /* Number of pages/bytes we'll compress at one time. */
411 #define LZO_UNC_PAGES   32
412 #define LZO_UNC_SIZE    (LZO_UNC_PAGES * PAGE_SIZE)
413
414 /* Number of pages/bytes we need for compressed data (worst case). */
415 #define LZO_CMP_PAGES   DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
416                                      LZO_HEADER, PAGE_SIZE)
417 #define LZO_CMP_SIZE    (LZO_CMP_PAGES * PAGE_SIZE)
418
419 /* Maximum number of threads for compression/decompression. */
420 #define LZO_THREADS     3
421
422 /* Maximum number of pages for read buffering. */
423 #define LZO_READ_PAGES  (MAP_PAGE_ENTRIES * 8)
424
425
426 /**
427  *      save_image - save the suspend image data
428  */
429
430 static int save_image(struct swap_map_handle *handle,
431                       struct snapshot_handle *snapshot,
432                       unsigned int nr_to_write)
433 {
434         unsigned int m;
435         int ret;
436         int nr_pages;
437         int err2;
438         struct bio *bio;
439         struct timeval start;
440         struct timeval stop;
441
442         printk(KERN_INFO "PM: Saving image data pages (%u pages) ...     ",
443                 nr_to_write);
444         m = nr_to_write / 100;
445         if (!m)
446                 m = 1;
447         nr_pages = 0;
448         bio = NULL;
449         do_gettimeofday(&start);
450         while (1) {
451                 ret = snapshot_read_next(snapshot);
452                 if (ret <= 0)
453                         break;
454                 ret = swap_write_page(handle, data_of(*snapshot), &bio);
455                 if (ret)
456                         break;
457                 if (!(nr_pages % m))
458                         printk(KERN_CONT "\b\b\b\b%3d%%", nr_pages / m);
459                 nr_pages++;
460         }
461         err2 = hib_wait_on_bio_chain(&bio);
462         do_gettimeofday(&stop);
463         if (!ret)
464                 ret = err2;
465         if (!ret)
466                 printk(KERN_CONT "\b\b\b\bdone\n");
467         else
468                 printk(KERN_CONT "\n");
469         swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
470         return ret;
471 }
472
473 /**
474  * Structure used for CRC32.
475  */
476 struct crc_data {
477         struct task_struct *thr;                  /* thread */
478         atomic_t ready;                           /* ready to start flag */
479         atomic_t stop;                            /* ready to stop flag */
480         unsigned run_threads;                     /* nr current threads */
481         wait_queue_head_t go;                     /* start crc update */
482         wait_queue_head_t done;                   /* crc update done */
483         u32 *crc32;                               /* points to handle's crc32 */
484         size_t *unc_len[LZO_THREADS];             /* uncompressed lengths */
485         unsigned char *unc[LZO_THREADS];          /* uncompressed data */
486 };
487
488 /**
489  * CRC32 update function that runs in its own thread.
490  */
491 static int crc32_threadfn(void *data)
492 {
493         struct crc_data *d = data;
494         unsigned i;
495
496         while (1) {
497                 wait_event(d->go, atomic_read(&d->ready) ||
498                                   kthread_should_stop());
499                 if (kthread_should_stop()) {
500                         d->thr = NULL;
501                         atomic_set(&d->stop, 1);
502                         wake_up(&d->done);
503                         break;
504                 }
505                 atomic_set(&d->ready, 0);
506
507                 for (i = 0; i < d->run_threads; i++)
508                         *d->crc32 = crc32_le(*d->crc32,
509                                              d->unc[i], *d->unc_len[i]);
510                 atomic_set(&d->stop, 1);
511                 wake_up(&d->done);
512         }
513         return 0;
514 }
515 /**
516  * Structure used for LZO data compression.
517  */
518 struct cmp_data {
519         struct task_struct *thr;                  /* thread */
520         atomic_t ready;                           /* ready to start flag */
521         atomic_t stop;                            /* ready to stop flag */
522         int ret;                                  /* return code */
523         wait_queue_head_t go;                     /* start compression */
524         wait_queue_head_t done;                   /* compression done */
525         size_t unc_len;                           /* uncompressed length */
526         size_t cmp_len;                           /* compressed length */
527         unsigned char unc[LZO_UNC_SIZE];          /* uncompressed buffer */
528         unsigned char cmp[LZO_CMP_SIZE];          /* compressed buffer */
529         unsigned char wrk[LZO1X_1_MEM_COMPRESS];  /* compression workspace */
530 };
531
532 /**
533  * Compression function that runs in its own thread.
534  */
535 static int lzo_compress_threadfn(void *data)
536 {
537         struct cmp_data *d = data;
538
539         while (1) {
540                 wait_event(d->go, atomic_read(&d->ready) ||
541                                   kthread_should_stop());
542                 if (kthread_should_stop()) {
543                         d->thr = NULL;
544                         d->ret = -1;
545                         atomic_set(&d->stop, 1);
546                         wake_up(&d->done);
547                         break;
548                 }
549                 atomic_set(&d->ready, 0);
550
551                 d->ret = lzo1x_1_compress(d->unc, d->unc_len,
552                                           d->cmp + LZO_HEADER, &d->cmp_len,
553                                           d->wrk);
554                 atomic_set(&d->stop, 1);
555                 wake_up(&d->done);
556         }
557         return 0;
558 }
559
560 /**
561  * save_image_lzo - Save the suspend image data compressed with LZO.
562  * @handle: Swap mam handle to use for saving the image.
563  * @snapshot: Image to read data from.
564  * @nr_to_write: Number of pages to save.
565  */
566 static int save_image_lzo(struct swap_map_handle *handle,
567                           struct snapshot_handle *snapshot,
568                           unsigned int nr_to_write)
569 {
570         unsigned int m;
571         int ret = 0;
572         int nr_pages;
573         int err2;
574         struct bio *bio;
575         struct timeval start;
576         struct timeval stop;
577         size_t off;
578         unsigned thr, run_threads, nr_threads;
579         unsigned char *page = NULL;
580         struct cmp_data *data = NULL;
581         struct crc_data *crc = NULL;
582
583         /*
584          * We'll limit the number of threads for compression to limit memory
585          * footprint.
586          */
587         nr_threads = num_online_cpus() - 1;
588         nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
589
590         page = (void *)__get_free_page(__GFP_WAIT | __GFP_HIGH);
591         if (!page) {
592                 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
593                 ret = -ENOMEM;
594                 goto out_clean;
595         }
596
597         data = vmalloc(sizeof(*data) * nr_threads);
598         if (!data) {
599                 printk(KERN_ERR "PM: Failed to allocate LZO data\n");
600                 ret = -ENOMEM;
601                 goto out_clean;
602         }
603         for (thr = 0; thr < nr_threads; thr++)
604                 memset(&data[thr], 0, offsetof(struct cmp_data, go));
605
606         crc = kmalloc(sizeof(*crc), GFP_KERNEL);
607         if (!crc) {
608                 printk(KERN_ERR "PM: Failed to allocate crc\n");
609                 ret = -ENOMEM;
610                 goto out_clean;
611         }
612         memset(crc, 0, offsetof(struct crc_data, go));
613
614         /*
615          * Start the compression threads.
616          */
617         for (thr = 0; thr < nr_threads; thr++) {
618                 init_waitqueue_head(&data[thr].go);
619                 init_waitqueue_head(&data[thr].done);
620
621                 data[thr].thr = kthread_run(lzo_compress_threadfn,
622                                             &data[thr],
623                                             "image_compress/%u", thr);
624                 if (IS_ERR(data[thr].thr)) {
625                         data[thr].thr = NULL;
626                         printk(KERN_ERR
627                                "PM: Cannot start compression threads\n");
628                         ret = -ENOMEM;
629                         goto out_clean;
630                 }
631         }
632
633         /*
634          * Adjust number of free pages after all allocations have been done.
635          * We don't want to run out of pages when writing.
636          */
637         handle->reqd_free_pages = reqd_free_pages();
638
639         /*
640          * Start the CRC32 thread.
641          */
642         init_waitqueue_head(&crc->go);
643         init_waitqueue_head(&crc->done);
644
645         handle->crc32 = 0;
646         crc->crc32 = &handle->crc32;
647         for (thr = 0; thr < nr_threads; thr++) {
648                 crc->unc[thr] = data[thr].unc;
649                 crc->unc_len[thr] = &data[thr].unc_len;
650         }
651
652         crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
653         if (IS_ERR(crc->thr)) {
654                 crc->thr = NULL;
655                 printk(KERN_ERR "PM: Cannot start CRC32 thread\n");
656                 ret = -ENOMEM;
657                 goto out_clean;
658         }
659
660         printk(KERN_INFO
661                 "PM: Using %u thread(s) for compression.\n"
662                 "PM: Compressing and saving image data (%u pages) ...     ",
663                 nr_threads, nr_to_write);
664         m = nr_to_write / 100;
665         if (!m)
666                 m = 1;
667         nr_pages = 0;
668         bio = NULL;
669         do_gettimeofday(&start);
670         for (;;) {
671                 for (thr = 0; thr < nr_threads; thr++) {
672                         for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) {
673                                 ret = snapshot_read_next(snapshot);
674                                 if (ret < 0)
675                                         goto out_finish;
676
677                                 if (!ret)
678                                         break;
679
680                                 memcpy(data[thr].unc + off,
681                                        data_of(*snapshot), PAGE_SIZE);
682
683                                 if (!(nr_pages % m))
684                                         printk(KERN_CONT "\b\b\b\b%3d%%",
685                                                nr_pages / m);
686                                 nr_pages++;
687                         }
688                         if (!off)
689                                 break;
690
691                         data[thr].unc_len = off;
692
693                         atomic_set(&data[thr].ready, 1);
694                         wake_up(&data[thr].go);
695                 }
696
697                 if (!thr)
698                         break;
699
700                 crc->run_threads = thr;
701                 atomic_set(&crc->ready, 1);
702                 wake_up(&crc->go);
703
704                 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
705                         wait_event(data[thr].done,
706                                    atomic_read(&data[thr].stop));
707                         atomic_set(&data[thr].stop, 0);
708
709                         ret = data[thr].ret;
710
711                         if (ret < 0) {
712                                 printk(KERN_ERR "PM: LZO compression failed\n");
713                                 goto out_finish;
714                         }
715
716                         if (unlikely(!data[thr].cmp_len ||
717                                      data[thr].cmp_len >
718                                      lzo1x_worst_compress(data[thr].unc_len))) {
719                                 printk(KERN_ERR
720                                        "PM: Invalid LZO compressed length\n");
721                                 ret = -1;
722                                 goto out_finish;
723                         }
724
725                         *(size_t *)data[thr].cmp = data[thr].cmp_len;
726
727                         /*
728                          * Given we are writing one page at a time to disk, we
729                          * copy that much from the buffer, although the last
730                          * bit will likely be smaller than full page. This is
731                          * OK - we saved the length of the compressed data, so
732                          * any garbage at the end will be discarded when we
733                          * read it.
734                          */
735                         for (off = 0;
736                              off < LZO_HEADER + data[thr].cmp_len;
737                              off += PAGE_SIZE) {
738                                 memcpy(page, data[thr].cmp + off, PAGE_SIZE);
739
740                                 ret = swap_write_page(handle, page, &bio);
741                                 if (ret)
742                                         goto out_finish;
743                         }
744                 }
745
746                 wait_event(crc->done, atomic_read(&crc->stop));
747                 atomic_set(&crc->stop, 0);
748         }
749
750 out_finish:
751         err2 = hib_wait_on_bio_chain(&bio);
752         do_gettimeofday(&stop);
753         if (!ret)
754                 ret = err2;
755         if (!ret) {
756                 printk(KERN_CONT "\b\b\b\bdone\n");
757         } else {
758                 printk(KERN_CONT "\n");
759         }
760         swsusp_show_speed(&start, &stop, nr_to_write, "Wrote");
761 out_clean:
762         if (crc) {
763                 if (crc->thr)
764                         kthread_stop(crc->thr);
765                 kfree(crc);
766         }
767         if (data) {
768                 for (thr = 0; thr < nr_threads; thr++)
769                         if (data[thr].thr)
770                                 kthread_stop(data[thr].thr);
771                 vfree(data);
772         }
773         if (page) free_page((unsigned long)page);
774
775         return ret;
776 }
777
778 /**
779  *      enough_swap - Make sure we have enough swap to save the image.
780  *
781  *      Returns TRUE or FALSE after checking the total amount of swap
782  *      space avaiable from the resume partition.
783  */
784
785 static int enough_swap(unsigned int nr_pages, unsigned int flags)
786 {
787         unsigned int free_swap = count_swap_pages(root_swap, 1);
788         unsigned int required;
789
790         pr_debug("PM: Free swap pages: %u\n", free_swap);
791
792         required = PAGES_FOR_IO + nr_pages;
793         return free_swap > required;
794 }
795
796 /**
797  *      swsusp_write - Write entire image and metadata.
798  *      @flags: flags to pass to the "boot" kernel in the image header
799  *
800  *      It is important _NOT_ to umount filesystems at this point. We want
801  *      them synced (in case something goes wrong) but we DO not want to mark
802  *      filesystem clean: it is not. (And it does not matter, if we resume
803  *      correctly, we'll mark system clean, anyway.)
804  */
805
806 int swsusp_write(unsigned int flags)
807 {
808         struct swap_map_handle handle;
809         struct snapshot_handle snapshot;
810         struct swsusp_info *header;
811         unsigned long pages;
812         int error;
813
814         pages = snapshot_get_image_size();
815         error = get_swap_writer(&handle);
816         if (error) {
817                 printk(KERN_ERR "PM: Cannot get swap writer\n");
818                 return error;
819         }
820         if (flags & SF_NOCOMPRESS_MODE) {
821                 if (!enough_swap(pages, flags)) {
822                         printk(KERN_ERR "PM: Not enough free swap\n");
823                         error = -ENOSPC;
824                         goto out_finish;
825                 }
826         }
827         memset(&snapshot, 0, sizeof(struct snapshot_handle));
828         error = snapshot_read_next(&snapshot);
829         if (error < PAGE_SIZE) {
830                 if (error >= 0)
831                         error = -EFAULT;
832
833                 goto out_finish;
834         }
835         header = (struct swsusp_info *)data_of(snapshot);
836         error = swap_write_page(&handle, header, NULL);
837         if (!error) {
838                 error = (flags & SF_NOCOMPRESS_MODE) ?
839                         save_image(&handle, &snapshot, pages - 1) :
840                         save_image_lzo(&handle, &snapshot, pages - 1);
841         }
842 out_finish:
843         error = swap_writer_finish(&handle, flags, error);
844         return error;
845 }
846
847 /**
848  *      The following functions allow us to read data using a swap map
849  *      in a file-alike way
850  */
851
852 static void release_swap_reader(struct swap_map_handle *handle)
853 {
854         struct swap_map_page_list *tmp;
855
856         while (handle->maps) {
857                 if (handle->maps->map)
858                         free_page((unsigned long)handle->maps->map);
859                 tmp = handle->maps;
860                 handle->maps = handle->maps->next;
861                 kfree(tmp);
862         }
863         handle->cur = NULL;
864 }
865
866 static int get_swap_reader(struct swap_map_handle *handle,
867                 unsigned int *flags_p)
868 {
869         int error;
870         struct swap_map_page_list *tmp, *last;
871         sector_t offset;
872
873         *flags_p = swsusp_header->flags;
874
875         if (!swsusp_header->image) /* how can this happen? */
876                 return -EINVAL;
877
878         handle->cur = NULL;
879         last = handle->maps = NULL;
880         offset = swsusp_header->image;
881         while (offset) {
882                 tmp = kmalloc(sizeof(*handle->maps), GFP_KERNEL);
883                 if (!tmp) {
884                         release_swap_reader(handle);
885                         return -ENOMEM;
886                 }
887                 memset(tmp, 0, sizeof(*tmp));
888                 if (!handle->maps)
889                         handle->maps = tmp;
890                 if (last)
891                         last->next = tmp;
892                 last = tmp;
893
894                 tmp->map = (struct swap_map_page *)
895                            __get_free_page(__GFP_WAIT | __GFP_HIGH);
896                 if (!tmp->map) {
897                         release_swap_reader(handle);
898                         return -ENOMEM;
899                 }
900
901                 error = hib_bio_read_page(offset, tmp->map, NULL);
902                 if (error) {
903                         release_swap_reader(handle);
904                         return error;
905                 }
906                 offset = tmp->map->next_swap;
907         }
908         handle->k = 0;
909         handle->cur = handle->maps->map;
910         return 0;
911 }
912
913 static int swap_read_page(struct swap_map_handle *handle, void *buf,
914                                 struct bio **bio_chain)
915 {
916         sector_t offset;
917         int error;
918         struct swap_map_page_list *tmp;
919
920         if (!handle->cur)
921                 return -EINVAL;
922         offset = handle->cur->entries[handle->k];
923         if (!offset)
924                 return -EFAULT;
925         error = hib_bio_read_page(offset, buf, bio_chain);
926         if (error)
927                 return error;
928         if (++handle->k >= MAP_PAGE_ENTRIES) {
929                 handle->k = 0;
930                 free_page((unsigned long)handle->maps->map);
931                 tmp = handle->maps;
932                 handle->maps = handle->maps->next;
933                 kfree(tmp);
934                 if (!handle->maps)
935                         release_swap_reader(handle);
936                 else
937                         handle->cur = handle->maps->map;
938         }
939         return error;
940 }
941
942 static int swap_reader_finish(struct swap_map_handle *handle)
943 {
944         release_swap_reader(handle);
945
946         return 0;
947 }
948
949 /**
950  *      load_image - load the image using the swap map handle
951  *      @handle and the snapshot handle @snapshot
952  *      (assume there are @nr_pages pages to load)
953  */
954
955 static int load_image(struct swap_map_handle *handle,
956                       struct snapshot_handle *snapshot,
957                       unsigned int nr_to_read)
958 {
959         unsigned int m;
960         int ret = 0;
961         struct timeval start;
962         struct timeval stop;
963         struct bio *bio;
964         int err2;
965         unsigned nr_pages;
966
967         printk(KERN_INFO "PM: Loading image data pages (%u pages) ...     ",
968                 nr_to_read);
969         m = nr_to_read / 100;
970         if (!m)
971                 m = 1;
972         nr_pages = 0;
973         bio = NULL;
974         do_gettimeofday(&start);
975         for ( ; ; ) {
976                 ret = snapshot_write_next(snapshot);
977                 if (ret <= 0)
978                         break;
979                 ret = swap_read_page(handle, data_of(*snapshot), &bio);
980                 if (ret)
981                         break;
982                 if (snapshot->sync_read)
983                         ret = hib_wait_on_bio_chain(&bio);
984                 if (ret)
985                         break;
986                 if (!(nr_pages % m))
987                         printk("\b\b\b\b%3d%%", nr_pages / m);
988                 nr_pages++;
989         }
990         err2 = hib_wait_on_bio_chain(&bio);
991         do_gettimeofday(&stop);
992         if (!ret)
993                 ret = err2;
994         if (!ret) {
995                 printk("\b\b\b\bdone\n");
996                 snapshot_write_finalize(snapshot);
997                 if (!snapshot_image_loaded(snapshot))
998                         ret = -ENODATA;
999         } else
1000                 printk("\n");
1001         swsusp_show_speed(&start, &stop, nr_to_read, "Read");
1002         return ret;
1003 }
1004
1005 /**
1006  * Structure used for LZO data decompression.
1007  */
1008 struct dec_data {
1009         struct task_struct *thr;                  /* thread */
1010         atomic_t ready;                           /* ready to start flag */
1011         atomic_t stop;                            /* ready to stop flag */
1012         int ret;                                  /* return code */
1013         wait_queue_head_t go;                     /* start decompression */
1014         wait_queue_head_t done;                   /* decompression done */
1015         size_t unc_len;                           /* uncompressed length */
1016         size_t cmp_len;                           /* compressed length */
1017         unsigned char unc[LZO_UNC_SIZE];          /* uncompressed buffer */
1018         unsigned char cmp[LZO_CMP_SIZE];          /* compressed buffer */
1019 };
1020
1021 /**
1022  * Deompression function that runs in its own thread.
1023  */
1024 static int lzo_decompress_threadfn(void *data)
1025 {
1026         struct dec_data *d = data;
1027
1028         while (1) {
1029                 wait_event(d->go, atomic_read(&d->ready) ||
1030                                   kthread_should_stop());
1031                 if (kthread_should_stop()) {
1032                         d->thr = NULL;
1033                         d->ret = -1;
1034                         atomic_set(&d->stop, 1);
1035                         wake_up(&d->done);
1036                         break;
1037                 }
1038                 atomic_set(&d->ready, 0);
1039
1040                 d->unc_len = LZO_UNC_SIZE;
1041                 d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len,
1042                                                d->unc, &d->unc_len);
1043                 atomic_set(&d->stop, 1);
1044                 wake_up(&d->done);
1045         }
1046         return 0;
1047 }
1048
1049 /**
1050  * load_image_lzo - Load compressed image data and decompress them with LZO.
1051  * @handle: Swap map handle to use for loading data.
1052  * @snapshot: Image to copy uncompressed data into.
1053  * @nr_to_read: Number of pages to load.
1054  */
1055 static int load_image_lzo(struct swap_map_handle *handle,
1056                           struct snapshot_handle *snapshot,
1057                           unsigned int nr_to_read)
1058 {
1059         unsigned int m;
1060         int ret = 0;
1061         int eof = 0;
1062         struct bio *bio;
1063         struct timeval start;
1064         struct timeval stop;
1065         unsigned nr_pages;
1066         size_t off;
1067         unsigned i, thr, run_threads, nr_threads;
1068         unsigned ring = 0, pg = 0, ring_size = 0,
1069                  have = 0, want, need, asked = 0;
1070         unsigned long read_pages;
1071         unsigned char **page = NULL;
1072         struct dec_data *data = NULL;
1073         struct crc_data *crc = NULL;
1074
1075         /*
1076          * We'll limit the number of threads for decompression to limit memory
1077          * footprint.
1078          */
1079         nr_threads = num_online_cpus() - 1;
1080         nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
1081
1082         page = vmalloc(sizeof(*page) * LZO_READ_PAGES);
1083         if (!page) {
1084                 printk(KERN_ERR "PM: Failed to allocate LZO page\n");
1085                 ret = -ENOMEM;
1086                 goto out_clean;
1087         }
1088
1089         data = vmalloc(sizeof(*data) * nr_threads);
1090         if (!data) {
1091                 printk(KERN_ERR "PM: Failed to allocate LZO data\n");
1092                 ret = -ENOMEM;
1093                 goto out_clean;
1094         }
1095         for (thr = 0; thr < nr_threads; thr++)
1096                 memset(&data[thr], 0, offsetof(struct dec_data, go));
1097
1098         crc = kmalloc(sizeof(*crc), GFP_KERNEL);
1099         if (!crc) {
1100                 printk(KERN_ERR "PM: Failed to allocate crc\n");
1101                 ret = -ENOMEM;
1102                 goto out_clean;
1103         }
1104         memset(crc, 0, offsetof(struct crc_data, go));
1105
1106         /*
1107          * Start the decompression threads.
1108          */
1109         for (thr = 0; thr < nr_threads; thr++) {
1110                 init_waitqueue_head(&data[thr].go);
1111                 init_waitqueue_head(&data[thr].done);
1112
1113                 data[thr].thr = kthread_run(lzo_decompress_threadfn,
1114                                             &data[thr],
1115                                             "image_decompress/%u", thr);
1116                 if (IS_ERR(data[thr].thr)) {
1117                         data[thr].thr = NULL;
1118                         printk(KERN_ERR
1119                                "PM: Cannot start decompression threads\n");
1120                         ret = -ENOMEM;
1121                         goto out_clean;
1122                 }
1123         }
1124
1125         /*
1126          * Start the CRC32 thread.
1127          */
1128         init_waitqueue_head(&crc->go);
1129         init_waitqueue_head(&crc->done);
1130
1131         handle->crc32 = 0;
1132         crc->crc32 = &handle->crc32;
1133         for (thr = 0; thr < nr_threads; thr++) {
1134                 crc->unc[thr] = data[thr].unc;
1135                 crc->unc_len[thr] = &data[thr].unc_len;
1136         }
1137
1138         crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
1139         if (IS_ERR(crc->thr)) {
1140                 crc->thr = NULL;
1141                 printk(KERN_ERR "PM: Cannot start CRC32 thread\n");
1142                 ret = -ENOMEM;
1143                 goto out_clean;
1144         }
1145
1146         /*
1147          * Adjust number of pages for read buffering, in case we are short.
1148          */
1149         read_pages = (nr_free_pages() - snapshot_get_image_size()) >> 1;
1150         read_pages = clamp_val(read_pages, LZO_CMP_PAGES, LZO_READ_PAGES);
1151
1152         for (i = 0; i < read_pages; i++) {
1153                 page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ?
1154                                                   __GFP_WAIT | __GFP_HIGH :
1155                                                   __GFP_WAIT);
1156                 if (!page[i]) {
1157                         if (i < LZO_CMP_PAGES) {
1158                                 ring_size = i;
1159                                 printk(KERN_ERR
1160                                        "PM: Failed to allocate LZO pages\n");
1161                                 ret = -ENOMEM;
1162                                 goto out_clean;
1163                         } else {
1164                                 break;
1165                         }
1166                 }
1167         }
1168         want = ring_size = i;
1169
1170         printk(KERN_INFO
1171                 "PM: Using %u thread(s) for decompression.\n"
1172                 "PM: Loading and decompressing image data (%u pages) ...     ",
1173                 nr_threads, nr_to_read);
1174         m = nr_to_read / 100;
1175         if (!m)
1176                 m = 1;
1177         nr_pages = 0;
1178         bio = NULL;
1179         do_gettimeofday(&start);
1180
1181         ret = snapshot_write_next(snapshot);
1182         if (ret <= 0)
1183                 goto out_finish;
1184
1185         for(;;) {
1186                 for (i = 0; !eof && i < want; i++) {
1187                         ret = swap_read_page(handle, page[ring], &bio);
1188                         if (ret) {
1189                                 /*
1190                                  * On real read error, finish. On end of data,
1191                                  * set EOF flag and just exit the read loop.
1192                                  */
1193                                 if (handle->cur &&
1194                                     handle->cur->entries[handle->k]) {
1195                                         goto out_finish;
1196                                 } else {
1197                                         eof = 1;
1198                                         break;
1199                                 }
1200                         }
1201                         if (++ring >= ring_size)
1202                                 ring = 0;
1203                 }
1204                 asked += i;
1205                 want -= i;
1206
1207                 /*
1208                  * We are out of data, wait for some more.
1209                  */
1210                 if (!have) {
1211                         if (!asked)
1212                                 break;
1213
1214                         ret = hib_wait_on_bio_chain(&bio);
1215                         if (ret)
1216                                 goto out_finish;
1217                         have += asked;
1218                         asked = 0;
1219                         if (eof)
1220                                 eof = 2;
1221                 }
1222
1223                 if (crc->run_threads) {
1224                         wait_event(crc->done, atomic_read(&crc->stop));
1225                         atomic_set(&crc->stop, 0);
1226                         crc->run_threads = 0;
1227                 }
1228
1229                 for (thr = 0; have && thr < nr_threads; thr++) {
1230                         data[thr].cmp_len = *(size_t *)page[pg];
1231                         if (unlikely(!data[thr].cmp_len ||
1232                                      data[thr].cmp_len >
1233                                      lzo1x_worst_compress(LZO_UNC_SIZE))) {
1234                                 printk(KERN_ERR
1235                                        "PM: Invalid LZO compressed length\n");
1236                                 ret = -1;
1237                                 goto out_finish;
1238                         }
1239
1240                         need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER,
1241                                             PAGE_SIZE);
1242                         if (need > have) {
1243                                 if (eof > 1) {
1244                                         ret = -1;
1245                                         goto out_finish;
1246                                 }
1247                                 break;
1248                         }
1249
1250                         for (off = 0;
1251                              off < LZO_HEADER + data[thr].cmp_len;
1252                              off += PAGE_SIZE) {
1253                                 memcpy(data[thr].cmp + off,
1254                                        page[pg], PAGE_SIZE);
1255                                 have--;
1256                                 want++;
1257                                 if (++pg >= ring_size)
1258                                         pg = 0;
1259                         }
1260
1261                         atomic_set(&data[thr].ready, 1);
1262                         wake_up(&data[thr].go);
1263                 }
1264
1265                 /*
1266                  * Wait for more data while we are decompressing.
1267                  */
1268                 if (have < LZO_CMP_PAGES && asked) {
1269                         ret = hib_wait_on_bio_chain(&bio);
1270                         if (ret)
1271                                 goto out_finish;
1272                         have += asked;
1273                         asked = 0;
1274                         if (eof)
1275                                 eof = 2;
1276                 }
1277
1278                 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
1279                         wait_event(data[thr].done,
1280                                    atomic_read(&data[thr].stop));
1281                         atomic_set(&data[thr].stop, 0);
1282
1283                         ret = data[thr].ret;
1284
1285                         if (ret < 0) {
1286                                 printk(KERN_ERR
1287                                        "PM: LZO decompression failed\n");
1288                                 goto out_finish;
1289                         }
1290
1291                         if (unlikely(!data[thr].unc_len ||
1292                                      data[thr].unc_len > LZO_UNC_SIZE ||
1293                                      data[thr].unc_len & (PAGE_SIZE - 1))) {
1294                                 printk(KERN_ERR
1295                                        "PM: Invalid LZO uncompressed length\n");
1296                                 ret = -1;
1297                                 goto out_finish;
1298                         }
1299
1300                         for (off = 0;
1301                              off < data[thr].unc_len; off += PAGE_SIZE) {
1302                                 memcpy(data_of(*snapshot),
1303                                        data[thr].unc + off, PAGE_SIZE);
1304
1305                                 if (!(nr_pages % m))
1306                                         printk("\b\b\b\b%3d%%", nr_pages / m);
1307                                 nr_pages++;
1308
1309                                 ret = snapshot_write_next(snapshot);
1310                                 if (ret <= 0) {
1311                                         crc->run_threads = thr + 1;
1312                                         atomic_set(&crc->ready, 1);
1313                                         wake_up(&crc->go);
1314                                         goto out_finish;
1315                                 }
1316                         }
1317                 }
1318
1319                 crc->run_threads = thr;
1320                 atomic_set(&crc->ready, 1);
1321                 wake_up(&crc->go);
1322         }
1323
1324 out_finish:
1325         if (crc->run_threads) {
1326                 wait_event(crc->done, atomic_read(&crc->stop));
1327                 atomic_set(&crc->stop, 0);
1328         }
1329         do_gettimeofday(&stop);
1330         if (!ret) {
1331                 printk("\b\b\b\bdone\n");
1332                 snapshot_write_finalize(snapshot);
1333                 if (!snapshot_image_loaded(snapshot))
1334                         ret = -ENODATA;
1335                 if (!ret) {
1336                         if (swsusp_header->flags & SF_CRC32_MODE) {
1337                                 if(handle->crc32 != swsusp_header->crc32) {
1338                                         printk(KERN_ERR
1339                                                "PM: Invalid image CRC32!\n");
1340                                         ret = -ENODATA;
1341                                 }
1342                         }
1343                 }
1344         } else
1345                 printk("\n");
1346         swsusp_show_speed(&start, &stop, nr_to_read, "Read");
1347 out_clean:
1348         for (i = 0; i < ring_size; i++)
1349                 free_page((unsigned long)page[i]);
1350         if (crc) {
1351                 if (crc->thr)
1352                         kthread_stop(crc->thr);
1353                 kfree(crc);
1354         }
1355         if (data) {
1356                 for (thr = 0; thr < nr_threads; thr++)
1357                         if (data[thr].thr)
1358                                 kthread_stop(data[thr].thr);
1359                 vfree(data);
1360         }
1361         if (page) vfree(page);
1362
1363         return ret;
1364 }
1365
1366 /**
1367  *      swsusp_read - read the hibernation image.
1368  *      @flags_p: flags passed by the "frozen" kernel in the image header should
1369  *                be written into this memory location
1370  */
1371
1372 int swsusp_read(unsigned int *flags_p)
1373 {
1374         int error;
1375         struct swap_map_handle handle;
1376         struct snapshot_handle snapshot;
1377         struct swsusp_info *header;
1378
1379         memset(&snapshot, 0, sizeof(struct snapshot_handle));
1380         error = snapshot_write_next(&snapshot);
1381         if (error < PAGE_SIZE)
1382                 return error < 0 ? error : -EFAULT;
1383         header = (struct swsusp_info *)data_of(snapshot);
1384         error = get_swap_reader(&handle, flags_p);
1385         if (error)
1386                 goto end;
1387         if (!error)
1388                 error = swap_read_page(&handle, header, NULL);
1389         if (!error) {
1390                 error = (*flags_p & SF_NOCOMPRESS_MODE) ?
1391                         load_image(&handle, &snapshot, header->pages - 1) :
1392                         load_image_lzo(&handle, &snapshot, header->pages - 1);
1393         }
1394         swap_reader_finish(&handle);
1395 end:
1396         if (!error)
1397                 pr_debug("PM: Image successfully loaded\n");
1398         else
1399                 pr_debug("PM: Error %d resuming\n", error);
1400         return error;
1401 }
1402
1403 /**
1404  *      swsusp_check - Check for swsusp signature in the resume device
1405  */
1406
1407 int swsusp_check(void)
1408 {
1409         int error;
1410
1411         hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device,
1412                                             FMODE_READ, NULL);
1413         if (!IS_ERR(hib_resume_bdev)) {
1414                 set_blocksize(hib_resume_bdev, PAGE_SIZE);
1415                 clear_page(swsusp_header);
1416                 error = hib_bio_read_page(swsusp_resume_block,
1417                                         swsusp_header, NULL);
1418                 if (error)
1419                         goto put;
1420
1421                 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
1422                         memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
1423                         /* Reset swap signature now */
1424                         error = hib_bio_write_page(swsusp_resume_block,
1425                                                 swsusp_header, NULL);
1426                 } else {
1427                         error = -EINVAL;
1428                 }
1429
1430 put:
1431                 if (error)
1432                         blkdev_put(hib_resume_bdev, FMODE_READ);
1433                 else
1434                         pr_debug("PM: Image signature found, resuming\n");
1435         } else {
1436                 error = PTR_ERR(hib_resume_bdev);
1437         }
1438
1439         if (error)
1440                 pr_debug("PM: Image not found (code %d)\n", error);
1441
1442         return error;
1443 }
1444
1445 /**
1446  *      swsusp_close - close swap device.
1447  */
1448
1449 void swsusp_close(fmode_t mode)
1450 {
1451         if (IS_ERR(hib_resume_bdev)) {
1452                 pr_debug("PM: Image device not initialised\n");
1453                 return;
1454         }
1455
1456         blkdev_put(hib_resume_bdev, mode);
1457 }
1458
1459 static int swsusp_header_init(void)
1460 {
1461         swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
1462         if (!swsusp_header)
1463                 panic("Could not allocate memory for swsusp_header\n");
1464         return 0;
1465 }
1466
1467 core_initcall(swsusp_header_init);