242
* write_area - Write data to given swap location.
243
* @fd: File handle of the resume partition
295
static int atomic_snapshot(int dev, int *in_suspend)
299
error = ioctl(dev, SNAPSHOT_CREATE_IMAGE, in_suspend);
300
if (error && errno == ENOTTY) {
301
report_unsupported_ioctl("SNAPSHOT_CREATE_IMAGE");
302
error = ioctl(dev, SNAPSHOT_ATOMIC_SNAPSHOT, in_suspend);
307
static inline int free_snapshot(int dev)
309
return ioctl(dev, SNAPSHOT_FREE, 0);
312
static int set_image_size(int dev, loff_t size)
316
error = ioctl(dev, SNAPSHOT_PREF_IMAGE_SIZE, size);
317
if (error && errno == ENOTTY) {
318
report_unsupported_ioctl("SNAPSHOT_PREF_IMAGE_SIZE");
319
error = ioctl(dev, SNAPSHOT_SET_IMAGE_SIZE, size);
324
static inline int suspend_to_ram(int dev)
326
return ioctl(dev, SNAPSHOT_S2RAM, 0);
329
static int platform_enter(int dev)
333
error = ioctl(dev, SNAPSHOT_POWER_OFF, 0);
334
if (error && errno == ENOTTY) {
335
report_unsupported_ioctl("SNAPSHOT_POWER_OFF");
336
error = ioctl(dev, SNAPSHOT_PMOPS, PMOPS_ENTER);
342
* alloc_swap - allocate a number of swap pages
343
* @dev: Swap device to use for allocations.
344
* @extents: Array of extents to track the allocations.
345
* @nr_extents: Number of extents already in the array.
346
* @size_p: Points to the number of bytes to allocate, used to
347
* return the number of allocated bytes.
349
* Allocate the number of swap pages sufficient for saving the number of
350
* bytes pointed to by @size_p. Use the array @extents to track the
351
* allocations. This array has to be page_size big and may already
352
* contain some initial elements (in that case @nr_extents must be the
353
* number of these elements).
354
* Each element of the array represents an area of allocated swap space.
355
* These areas may be extended when swap pages that can be added to them
356
* are found. They also can be merged with one another.
357
* The function returns when the requested amount of swap space is
358
* allocated or if there is no room for more extents. In the latter case
359
* the last extent created is put at the end of the array and may be passed
360
* to alloc_swap() as the initial extent when it is invoked next time.
363
alloc_swap(int dev, struct extent *extents, int nr_extents, loff_t *size_p)
365
const int max_extents = page_size / sizeof(struct extent) - 1;
366
loff_t size, total_size, offset;
368
total_size = *size_p;
369
if (nr_extents <= 0) {
370
offset = get_swap_page(dev);
373
extents->start = offset;
374
extents->end = offset + page_size;
380
while (size < total_size && nr_extents <= max_extents) {
383
offset = get_swap_page(dev);
386
/* Check if we have a matching extent. */
395
if (offset == ext->start - page_size) {
397
/* Check if we can merge extents */
398
if (k > 0 && extents[k-1].end == offset) {
399
extents[k-1].end = ext->end;
400
/* Pull the 'later' extents forward */
401
memmove(ext, ext + 1,
402
(nr_extents - k - 1) *
408
} else if (offset == ext->end) {
409
ext->end += page_size;
410
/* Check if we can merge extents */
411
if (k + 1 < nr_extents
412
&& ext->end == extents[k+1].start) {
413
ext->end = extents[k+1].end;
414
/* Pull the 'later' extents forward */
415
memmove(ext + 1, ext + 2,
416
(nr_extents - k - 2) *
422
} else if (offset > ext->end) {
425
/* This means i == j + 1 */
434
/* offset < ext->start - page_size */
439
/* No match. Create a new extent. */
442
if (nr_extents < max_extents) {
445
* We want to always replace the extent 'i' with
448
if (offset > ext->end) {
452
/* Push the 'later' extents backwards. */
453
memmove(ext + 1, ext,
454
(nr_extents - i) * sizeof(*ext));
456
ext = extents + nr_extents;
459
ext->end = offset + page_size;
469
* write_page - Write page_size data to given swap location.
470
* @fd: File handle of the resume partition.
244
471
* @buf: Pointer to the area we're writing.
245
* @offset: Offset of the swap page we're writing to
246
* @size: The number of bytes to save
472
* @offset: Offset of the swap page we're writing to.
249
static int write_area(int fd, void *buf, loff_t offset, unsigned int size)
474
static int write_page(int fd, void *buf, loff_t offset)
255
if (lseek(fd, offset, SEEK_SET) == offset)
256
cnt = write(fd, buf, size);
260
res = cnt < 0 ? cnt : -EIO;
482
if (lseek64(fd, offset, SEEK_SET) == offset)
483
cnt = write(fd, buf, page_size);
484
if (cnt != page_size)
266
* The swap_map_handle structure is used for handling swap in
490
* The swap_writer structure is used for handling swap in a file-alike way.
492
* @extents: Array of extents used for trackig swap allocations. It is
493
* page_size bytes large and holds at most
494
* (page_size / sizeof(struct extent) - 1) extents. The last slot
495
* is used to hold the extent that will be used as an initial one
496
* for the next batch of allocations.
498
* @nr_extents: Number of entries in @extents actually used.
500
* @cur_extent: The extent currently used as the source of swap pages.
502
* @cur_extent_idx: The index of @cur_extent.
504
* @cur_offset: The offset of the swap page that will be used next.
506
* @swap_needed: The amount of swap needed for saving the image.
508
* @written_data: The amount of data actually saved.
510
* @extents_spc: The swap page to which to save @extents.
512
* @buffer: Buffer used for storing image data pages.
514
* @write_buffer: If compression is used, the compressed contents of
515
* @buffer are stored here. Otherwise, it is equal to
518
* @page_ptr: Address to write the next image page to.
520
* @dev: Snapshot device handle used for reading image pages and
523
* @fd: File handle associated with the swap.
525
* @ctx: Used for checksum computing, if so configured.
527
* @lzo_work_buffer: Work buffer used for compression.
529
* @encrypt_buffer: Buffer for storing encrypted data (page_size bytes).
531
* @encrypt_ptr: Address to store the next encrypted page at.
270
struct swap_map_handle {
273
struct swap_area *areas;
274
unsigned short areas_per_page;
276
struct swap_area cur_area;
277
unsigned int cur_alloc;
534
struct extent *extents;
536
struct extent *cur_extent;
281
546
struct md5_ctx ctx;
282
#ifdef CONFIG_COMPRESS
283
547
void *lzo_work_buffer;
285
#ifdef CONFIG_ENCRYPT
286
unsigned char *encrypt_buffer;
548
void *encrypt_buffer;
291
init_swap_writer(struct swap_map_handle *handle, int dev, int fd, void *buf)
299
handle->page_buffer = buf;
302
handle->write_buffer = buf;
305
#ifdef CONFIG_COMPRESS
306
handle->lzo_work_buffer = buf;
307
buf += LZO1X_1_MEM_COMPRESS;
310
#ifdef CONFIG_ENCRYPT
311
handle->encrypt_buffer = buf;
314
memset(handle->areas, 0, page_size);
315
handle->areas_per_page = (page_size - sizeof(loff_t)) /
316
sizeof(struct swap_area);
317
handle->next_swap = (loff_t *)(handle->areas + handle->areas_per_page);
319
handle->cur_swap = get_swap_page(dev);
320
if (!handle->cur_swap)
322
handle->cur_area.offset = get_swap_page(dev);
323
if (!handle->cur_area.offset)
325
handle->cur_area.size = 0;
326
handle->cur_alloc = page_size;
553
* free_swap_writer - free memory allocated for saving the image
554
* @handle: Structure containing pointers to memory buffers to free.
556
static void free_swap_writer(struct swap_writer *handle)
558
if (handle->write_buffer != handle->buffer)
559
freemem(handle->write_buffer);
561
freemem(handle->lzo_work_buffer);
562
if (handle->encrypt_buffer)
563
freemem(handle->encrypt_buffer);
564
freemem(handle->buffer);
565
freemem(handle->extents);
569
* init_swap_writer - initialize the structure used for saving the image
570
* @handle: Structure to initialize.
571
* @dev: Special device file to read image pages from.
572
* @fd: File descriptor associated with the swap.
574
* It doesn't preallocate swap, so preallocate_swap() has to be called on
575
* @handle after this.
577
static int init_swap_writer(struct swap_writer *handle, int dev, int fd, int in)
580
unsigned int write_buf_size = 0;
582
handle->extents = getmem(page_size);
584
handle->buffer = getmem(buffer_size);
585
handle->page_ptr = handle->buffer;
588
handle->encrypt_buffer = getmem(encrypt_buf_size);
589
handle->encrypt_ptr = handle->encrypt_buffer;
591
handle->encrypt_buffer = NULL;
595
handle->lzo_work_buffer = getmem(LZO1X_1_MEM_COMPRESS);
596
write_buf_size = compress_buf_size;
599
(WRITE_BUFFERS - 1) * compress_buf_size;
602
if (write_buf_size > 0)
603
handle->write_buffer = getmem(write_buf_size);
604
else if (use_threads)
605
handle->write_buffer = getmem(buffer_size * WRITE_BUFFERS);
607
handle->write_buffer = handle->buffer;
329
609
handle->dev = dev;
332
if (compute_checksum)
611
handle->input = (in >= 0) ? in : dev;
612
handle->written_data = 0;
614
memset(handle->extents, 0, page_size);
615
handle->nr_extents = 0;
616
offset = get_swap_page(dev);
618
free_swap_writer(handle);
621
handle->extents_spc = offset;
623
if (compute_checksum || verify_image)
333
624
md5_init_ctx(&handle->ctx);
338
static int prepare(struct swap_map_handle *handle, int disp)
340
struct buf_block *block;
341
void *buf = handle->page_buffer;
343
block = (struct buf_block *)(handle->write_buffer + disp);
630
* preallocate_swap - use alloc_swap() to preallocate the number of pages
631
* given by @handle->swap_needed
632
* @handle: Pointer to the structure in which to store information
633
* about the preallocated swap pool.
635
* Returns the offset of the first swap page available from the
638
static loff_t preallocate_swap(struct swap_writer *handle)
640
const int max = page_size / sizeof(struct extent) - 1;
644
if (handle->swap_needed < page_size)
646
size = handle->swap_needed;
647
if (do_compress && size > page_size)
649
nr_extents = alloc_swap(handle->dev, handle->extents,
650
handle->nr_extents, &size);
653
handle->nr_extents = nr_extents < max ? nr_extents : max;
654
handle->cur_extent = handle->extents;
655
handle->cur_extent_idx = 0;
656
handle->cur_offset = handle->cur_extent->start;
657
return handle->cur_offset;
661
* save_extents - save the array of extents
662
* handle: Structure holding the pointer to the array of extents etc.
663
* finish: If set, the last element of the extents array has to be filled
666
* Save the buffer (page) holding the array of extents to the swap
667
* location pointed to by @handle->extents_spc (this must be allocated
668
* earlier). Before saving the last element of the array is used to store
669
* the swap offset of the next extents page (we allocate a swap page for
672
static int save_extents(struct swap_writer *handle, int finish)
678
struct extent *last_extent;
680
offset = get_swap_page(handle->dev);
683
last_extent = handle->extents +
684
page_size / sizeof(struct extent) - 1;
685
last_extent->start = offset;
687
error = write_page(handle->fd, handle->extents, handle->extents_spc);
688
handle->extents_spc = offset;
693
* next_swap_page - take one swap page out of the pool allocated using
694
* alloc_swap() before
695
* @handle: Pointer to the structure containing information about
696
* the preallocated swap pool.
698
static loff_t next_swap_page(struct swap_writer *handle)
702
handle->cur_offset += page_size;
703
if (handle->cur_offset < handle->cur_extent->end)
704
return handle->cur_offset;
705
/* We have exhausted the current extent. Forward to the next one */
706
handle->cur_extent++;
707
handle->cur_extent_idx++;
708
if (handle->cur_extent_idx < handle->nr_extents) {
709
handle->cur_offset = handle->cur_extent->start;
710
return handle->cur_offset;
712
/* No more extents. Is there anything to pass to alloc_swap()? */
713
if (handle->cur_extent->start < handle->cur_extent->end) {
714
ext = *handle->cur_extent;
715
memset(handle->cur_extent, 0, sizeof(struct extent));
716
handle->nr_extents = 1;
718
memset(&ext, 0, sizeof(struct extent));
719
handle->nr_extents = 0;
721
if (save_extents(handle, 0))
723
memset(handle->extents, 0, page_size);
724
*handle->extents = ext;
725
return preallocate_swap(handle);
729
* save_page - save one page of data to the swap
730
* @handle: Pointer to the structure containing information about
732
* @src: Pointer to the data.
734
static int save_page(struct swap_writer *handle, void *src)
739
offset = next_swap_page(handle);
742
error = write_page(handle->fd, src, offset);
745
handle->swap_needed -= page_size;
746
handle->written_data += page_size;
750
#ifdef CONFIG_THREADS
752
* If threads are used for saving the image with compression and encryption,
753
* there are three of them.
755
* The main one reads image pages from the kernel and puts them into a work
756
* buffer. When the work buffer is full, it gets compressed, but that's not an
757
* in-place compression, so the result has to be stored somewhere else. There
758
* are four so-called "write" buffers for that and the first empty "write"
759
* buffer is used as the target. If all of the "write" buffers are full, the
760
* thread has to wait (see the rules below). Otherwise, after placing the
761
* (compressed) contents of the work buffer into a "write" buffer, the main
762
* thread regards the work buffer as empty and starts to read more image pages
765
* The second thread (call it the "move" thread) encrypts the contents of the
766
* "write" buffers, one buffer at a time. It really encrypts individual pages
767
* and the encryption is not in-place, too. The encrypted pages of data are
768
* placed in yet another buffer (call it the "encrypt" buffer) until it's full,
769
* in which case the "move" thread has to wait. Of course, it also has to wait
770
* for data from the main thread if all of the "write" buffers are empty.
771
* After encrypting an entire "write" buffer, the "move" thread progresses to
772
* the next "write" buffer, in a round-robin manner.
774
* The synchronization between the main thread and the "move" thread is done
775
* with the help of two index variables, move_start and move_end. �The rule
777
* (1) the main thread can only put data into write_buffers[move_start],
778
* (2) after putting data into write_buffers[move_start], the main thread
779
* increases move_start, modulo the number of "write" buffers, but
780
* move_start cannot be modified as long as the _next_ "write" buffer is
781
* write_buffers[move_end] (the thread has to wait if that happens),
782
* (3) the "move" thread can only read data from write_buffers[move_end] and
783
* only if move_end != move_start (it has to wait if that's not the case),
784
* (4) after reading data from write_buffers[move_end], the "move" thread
785
* increases move_end, modulo the number of "write" buffers.
786
* This way, move_end always "follows" move_start and the threads don't access
787
* the same buffer at any time.
789
* The third thread (call it the "save" thread) reads (encrypted) pages of data
790
* from the "encrypt" buffer and writes them out to the swap. This is done if
791
* there are some pages to write in the "encrypt" buffer, otherwise the "save"
792
* thread has to wait for the "move" thread to put more pages in there.
794
* The synchronization between the "move" thread and the "save" thread is done
795
* with the help of two pointers, save_start and save_end, where save_start
796
* points to the first empty page and save_end points to the last data page
797
* that hasn't been written out yet. Thus, the rule is:
798
* (1) the "move" thread can only put data into the page pointed to by
800
* (2) after putting data into the page pointed to by save_start, the "move"
801
* thread increases save_start, modulo the number of pages in the buffer,
802
* provided that the _next_ page is not the one pointed to by save_end (it
803
* has to wait if that happens),
804
* (3) the "save" thread can only read from the page pointed to by save_end,
805
* as long as save_end != save_start (it has to wait if the two pointers
807
* (4) after writing data from the page pointed to by save_end, the "save"
808
* thread increases save_end, modulo the number of pages in the buffer.
809
* IOW, the "encrypt" buffer is handled as a typical circular buffer with one
810
* producer (the "move" thread) and one consumer (the "save" thread).
812
* If encryption is not used, the "save" thread is not started and the "move"
813
* thread writes data to the swap directly out of the "write" buffers.
817
static pthread_mutex_t finish_mutex;
818
static pthread_cond_t finish_cond;
820
static char *encrypt_buf;
821
static char *save_start, *save_end;
822
static pthread_mutex_t save_mutex;
823
static pthread_cond_t save_cond;
824
static pthread_t save_th;
826
struct write_buffer {
831
static struct write_buffer write_buffers[WRITE_BUFFERS];
832
static int move_start, move_end;
833
static pthread_mutex_t move_mutex;
834
static pthread_cond_t move_cond;
835
static pthread_t move_th;
839
static char *save_inc(char *ptr)
842
(((ptr - encrypt_buf) + page_size) % encrypt_buf_size);
845
static int move_inc(int index)
847
return (index + 1) % WRITE_BUFFERS;
850
static int wait_for_finish(void)
852
pthread_mutex_lock(&finish_mutex);
853
while((save_end != save_start || move_start != move_end) && !save_ret)
854
pthread_cond_wait(&finish_cond, &finish_mutex);
855
pthread_mutex_unlock(&finish_mutex);
859
static void *save_thread(void *arg)
861
struct swap_writer *handle = arg;
865
/* Wait until there is a buffer ready for processing. */
866
pthread_mutex_lock(&save_mutex);
867
while(save_end == save_start && !save_ret)
868
pthread_cond_wait(&save_cond, &save_mutex);
869
pthread_mutex_unlock(&save_mutex);
874
error = save_page(handle, save_end);
876
pthread_mutex_lock(&finish_mutex);
879
pthread_mutex_unlock(&finish_mutex);
880
pthread_cond_signal(&move_cond);
881
pthread_cond_signal(&save_cond);
882
pthread_cond_signal(&finish_cond);
886
/* Go to the next page */
887
pthread_mutex_lock(&finish_mutex);
888
pthread_mutex_lock(&save_mutex);
889
save_end = save_inc(save_end);
890
pthread_mutex_unlock(&save_mutex);
891
pthread_mutex_unlock(&finish_mutex);
893
pthread_cond_signal(&save_cond);
894
pthread_cond_signal(&finish_cond);
900
#ifdef CONFIG_ENCRYPT
902
static void encrypt_and_save_buffer(void)
905
ssize_t buf_size, moved_size;
908
* The buffer to process is at write_buffers[move_end].start and the
909
* size of it is write_buffers[move_end].size .
911
src = write_buffers[move_end].start;
912
buf_size = write_buffers[move_end].size;
918
/* Encrypt page_size of data. */
919
error = gcry_cipher_encrypt(cipher_handle,
920
save_start, page_size,
923
pthread_mutex_lock(&finish_mutex);
926
pthread_mutex_unlock(&finish_mutex);
927
pthread_cond_signal(&move_cond);
928
pthread_cond_signal(&save_cond);
929
pthread_cond_signal(&finish_cond);
933
moved_size += page_size;
936
pthread_mutex_lock(&save_mutex);
937
next_start = save_inc(save_start);
938
while (next_start == save_end && !save_ret)
939
pthread_cond_wait(&save_cond, &save_mutex);
940
save_start = next_start;
941
pthread_mutex_unlock(&save_mutex);
943
pthread_cond_signal(&save_cond);
944
} while (moved_size < buf_size && !save_ret);
947
#else /* !CONFIG_ENCRYPT */
949
static inline void encrypt_and_save_buffer(void) {}
951
#endif /* !CONFIG_ENCRYPT */
953
static void save_buffer(struct swap_writer *handle)
959
* The buffer to process is at write_buffers[move_end].start and the
960
* size of it is write_buffers[move_end].size .
962
src = write_buffers[move_end].start;
963
size = write_buffers[move_end].size;
965
int error = save_page(handle, src);
967
pthread_mutex_lock(&finish_mutex);
970
pthread_mutex_unlock(&finish_mutex);
971
pthread_cond_signal(&move_cond);
972
pthread_cond_signal(&finish_cond);
980
static void *move_thread(void *arg)
982
struct swap_writer *handle = arg;
985
/* Wait until there is a buffer ready for processing. */
986
pthread_mutex_lock(&move_mutex);
987
while(move_end == move_start && !save_ret)
988
pthread_cond_wait(&move_cond, &move_mutex);
989
pthread_mutex_unlock(&move_mutex);
995
encrypt_and_save_buffer();
1002
/* Tell the reader thread that we have processed the buffer */
1003
pthread_mutex_lock(&finish_mutex);
1004
pthread_mutex_lock(&move_mutex);
1005
move_end = move_inc(move_end);
1006
pthread_mutex_unlock(&move_mutex);
1007
pthread_mutex_unlock(&finish_mutex);
1009
pthread_cond_signal(&move_cond);
1010
pthread_cond_signal(&finish_cond);
1016
static inline void *current_write_buffer(void)
1018
return write_buffers[move_start].start;
1021
static int prepare_next_write_buffer(ssize_t size)
1025
/* Move to the next buffer and signal that the current one is ready*/
1026
write_buffers[move_start].size = size;
1028
pthread_mutex_lock(&move_mutex);
1029
next_start = move_inc(move_start);
1030
while (next_start == move_end && !save_ret)
1031
pthread_cond_wait(&move_cond, &move_mutex);
1032
move_start = next_start;
1033
pthread_mutex_unlock(&move_mutex);
1035
pthread_cond_signal(&move_cond);
1040
static void start_threads(struct swap_writer *handle)
1043
unsigned int write_buf_size;
1047
encrypt_buf = handle->encrypt_buffer;
1048
save_start = encrypt_buf;
1049
save_end = save_start;
1051
write_buf_size = do_compress ? compress_buf_size : buffer_size;
1052
write_buf = handle->write_buffer;
1053
for (j = 0; j < WRITE_BUFFERS; j++) {
1054
write_buffers[j].start = write_buf;
1055
write_buf += write_buf_size;
1058
move_end = move_start;
1061
error = pthread_mutex_init(&save_mutex, NULL);
1063
perror("pthread_mutex_init() failed:");
1066
error = pthread_cond_init(&save_cond, NULL);
1068
perror("pthread_cond_init() failed:");
1069
goto Destroy_save_mutex;
1073
error = pthread_mutex_init(&move_mutex, NULL);
1075
perror("pthread_mutex_init() failed:");
1076
goto Destroy_save_cond;
1078
error = pthread_cond_init(&move_cond, NULL);
1080
perror("pthread_cond_init() failed:");
1081
goto Destroy_move_mutex;
1085
error = pthread_create(&save_th, NULL, save_thread, handle);
1087
perror("pthread_create() failed:");
1088
goto Destroy_move_cond;
1092
error = pthread_create(&move_th, NULL, move_thread, handle);
1094
perror("pthread_create() failed:");
1095
goto Stop_save_thread;
1098
error = pthread_mutex_init(&finish_mutex, NULL);
1100
perror("pthread_mutex_init() failed:");
1101
goto Stop_move_thread;
1103
error = pthread_cond_init(&finish_cond, NULL);
1105
perror("pthread_cond_init() failed:");
1106
goto Destroy_finish_mutex;
1111
Destroy_finish_mutex:
1112
pthread_mutex_destroy(&finish_mutex);
1115
save_ret = FORCE_EXIT;
1116
pthread_cond_signal(&move_cond);
1117
pthread_join(move_th, NULL);
1121
save_ret = FORCE_EXIT;
1122
pthread_cond_signal(&save_cond);
1123
pthread_join(save_th, NULL);
1127
pthread_cond_destroy(&move_cond);
1129
pthread_mutex_destroy(&move_mutex);
1133
pthread_cond_destroy(&save_cond);
1136
pthread_mutex_destroy(&save_mutex);
1142
static void stop_threads(void)
1144
pthread_mutex_lock(&finish_mutex);
1146
save_ret = FORCE_EXIT;
1147
pthread_mutex_unlock(&finish_mutex);
1149
pthread_cond_destroy(&finish_cond);
1150
pthread_mutex_destroy(&finish_mutex);
1152
pthread_cond_signal(&move_cond);
1153
pthread_join(move_th, NULL);
1155
pthread_cond_signal(&save_cond);
1156
pthread_join(save_th, NULL);
1159
pthread_cond_destroy(&move_cond);
1160
pthread_mutex_destroy(&move_mutex);
1163
pthread_cond_destroy(&save_cond);
1164
pthread_mutex_destroy(&save_mutex);
1168
#else /* !CONFIG_THREADS */
1170
static inline int wait_for_finish(void) { return -ENOSYS; }
1171
static inline void *current_write_buffer(void) { return NULL; }
1172
static inline int prepare_next_write_buffer(ssize_t size)
1177
static inline void start_threads(struct swap_writer *handle) { (void)handle; }
1178
static inline void stop_threads(void) {}
1180
#endif /* !CONFIG_THREADS */
1183
* encrypt_and_save_page - encrypt a page of data and write it to the swap
1185
static int encrypt_and_save_page(struct swap_writer *handle, void *src)
1187
#ifdef CONFIG_ENCRYPT
1189
int error = gcry_cipher_encrypt(cipher_handle,
1190
handle->encrypt_ptr, page_size, src, page_size);
1193
src = handle->encrypt_ptr;
1194
handle->encrypt_ptr += page_size;
1195
if (handle->encrypt_ptr - handle->encrypt_buffer
1196
>= encrypt_buf_size)
1197
handle->encrypt_ptr = handle->encrypt_buffer;
1200
return save_page(handle, src);
1204
* flush_buffer - flush data stored in the buffer to the swap
1206
static int flush_buffer(struct swap_writer *handle)
1212
/* Check if there is anything to do */
1213
if (handle->page_ptr <= handle->buffer)
1216
size = handle->page_ptr - handle->buffer;
1217
if (compute_checksum || verify_image)
1218
md5_process_block(handle->buffer, size, &handle->ctx);
1220
src = use_threads ? current_write_buffer() : handle->write_buffer;
1222
/* Compress the buffer, if necessary */
344
1224
#ifdef CONFIG_COMPRESS
1225
struct buf_block *block = (struct buf_block *)src;
348
lzo1x_1_compress(buf, page_size, (lzo_bytep)block->data, &cnt,
349
handle->lzo_work_buffer);
1228
lzo1x_1_compress(handle->buffer, size,
1229
(lzo_bytep)block->data, &cnt,
1230
handle->lzo_work_buffer);
350
1231
block->size = cnt;
351
/* Sanity check, should not trigger */
352
if (cnt + sizeof(short) > max_block_size)
354
" Data size too large after compression!\n");
355
/* We have added a sizeof(short) field to the data */
356
compr_diff += page_size - cnt - sizeof(short);
357
return cnt + sizeof(short);
360
memcpy(block, buf, page_size);
364
static int try_get_more_swap(struct swap_map_handle *handle)
368
while (handle->cur_area.size > handle->cur_alloc) {
369
offset = get_swap_page(handle->dev);
372
if (offset == handle->cur_area.offset + handle->cur_alloc) {
373
handle->cur_alloc += page_size;
375
handle->cur_area.offset = offset;
376
handle->cur_alloc = page_size;
382
static int flush_buffer(struct swap_map_handle *handle)
384
void *src = handle->write_buffer;
387
#ifdef CONFIG_ENCRYPT
389
error = gcry_cipher_encrypt(cipher_handle,
390
handle->encrypt_buffer, handle->cur_area.size,
391
src, handle->cur_area.size);
392
src = handle->encrypt_buffer;
396
error = write_area(handle->fd, src, handle->cur_area.offset,
397
handle->cur_area.size);
400
handle->areas[handle->k].offset = handle->cur_area.offset;
401
handle->areas[handle->k].size = handle->cur_area.size;
405
static int swap_write_page(struct swap_map_handle *handle)
410
if (compute_checksum)
411
md5_process_block(handle->page_buffer, page_size, &handle->ctx);
413
if (!handle->cur_area.size) {
414
/* No data in the write buffer */
415
handle->cur_area.size = prepare(handle, 0);
416
return try_get_more_swap(handle);
419
if (handle->cur_alloc + max_block_size <= buffer_size) {
420
if (handle->cur_area.size + max_block_size <= handle->cur_alloc) {
421
handle->cur_area.size += prepare(handle, handle->cur_area.size);
424
offset = get_swap_page(handle->dev);
427
if (offset == handle->cur_area.offset + handle->cur_alloc) {
428
handle->cur_alloc += page_size;
429
handle->cur_area.size += prepare(handle, handle->cur_area.size);
430
return try_get_more_swap(handle);
434
/* The write buffer is full or the offset doesn't fit. Flush the buffer */
435
error = flush_buffer(handle);
439
/* The write buffer has been flushed. Fill it from the start */
441
offset = get_swap_page(handle->dev);
445
handle->cur_area.offset = offset;
446
handle->cur_alloc = page_size;
447
handle->cur_area.size = prepare(handle, 0);
448
error = try_get_more_swap(handle);
451
if (++handle->k >= handle->areas_per_page) {
452
offset = get_swap_page(handle->dev);
455
*handle->next_swap = offset;
456
error = write_area(handle->fd, handle->areas, handle->cur_swap,
1232
size = cnt + sizeof(size_t);
1234
} else if (use_threads) {
1235
memcpy(src, handle->buffer, size);
1239
return prepare_next_write_buffer(size);
1242
* If there's no compression and threads are not used, handle->buffer is
1243
* equal to handle->write_buffer. In that case, the data are taken
1244
* directly out of handle->buffer.
1247
error = encrypt_and_save_page(handle, src);
460
memset(handle->areas, 0, page_size);
461
handle->cur_swap = offset;
467
static inline int flush_swap_writer(struct swap_map_handle *handle)
471
if (!handle->cur_area.offset || !handle->cur_swap)
474
/* Flush the write buffer */
475
error = flush_buffer(handle);
477
/* Save the last swap map page */
478
error = write_area(handle->fd, handle->areas, handle->cur_swap,
485
* save_image - save the suspend image data
1258
* save_image - save the hibernation image data
488
static int save_image(struct swap_map_handle *handle,
489
unsigned int nr_pages)
1260
static int save_image(struct swap_writer *handle, unsigned int nr_pages)
491
1262
unsigned int m, writeout_rate;
492
int ret, abort_possible;
493
1264
struct termios newtrm, savedtrm;
1265
int abort_possible, key, error = 0;
495
1266
char message[SPLASH_GENERIC_MESSAGE_SIZE];
497
1268
/* Switch the state of the terminal so that we can read the keyboard
615
1415
* write_image - Write entire image and metadata.
1416
* @snapshot_fd: File handle of the snapshot device
1417
* @resume_fd: File handle of the swap device used for image saving
1418
* @test_fd: (Optional) File handle of a file to read the image from
1420
* If @test_fd is not negative, the function works in the test mode in
1421
* which the image is read from a regular file instead of the snapshot
618
int write_image(int snapshot_fd, int resume_fd)
1424
static int write_image(int snapshot_fd, int resume_fd, int test_fd)
620
static struct swap_map_handle handle;
621
struct swsusp_info *header = mem_pool;
1426
static struct swap_writer handle;
1427
struct image_header_info *header;
624
struct timeval begin;
1431
unsigned long nr_pages = 0;
1432
int error, test_mode = (test_fd >= 0);
1433
struct timeval begin;
626
1435
printf("%s: System snapshot ready. Preparing to write\n", my_name);
627
start = get_swap_page(snapshot_fd);
1436
/* Allocate a swap page for the additional "userland" header */
1437
start = alloc_swap_page(snapshot_fd, 1);
630
error = read(snapshot_fd, header, page_size);
631
if (error < (int)page_size)
632
return error < 0 ? error : -EFAULT;
633
printf("%s: Image size: %lu kilobytes\n", my_name, header->size / 1024);
634
if (!enough_swap(snapshot_fd, header->size) && !do_compress) {
1441
header = getmem(page_size);
1442
memset(header, 0, page_size);
1444
error = init_swap_writer(&handle, snapshot_fd, resume_fd, test_fd);
1448
image_size = test_mode ? test_image_size : get_image_size(snapshot_fd);
1449
if (image_size > 0) {
1450
nr_pages = (unsigned long)((image_size + page_size - 1) /
1454
* The kernel doesn't allow us to get the image size via ioctl,
1455
* so we need to read it from the image header.
1457
struct swsusp_info *image_header;
1461
* Do it in such a way that save_image() will believe it has
1462
* already read the header page.
1464
image_header = handle.page_ptr;
1465
ret = read(snapshot_fd, image_header, page_size);
1466
if (ret < page_size) {
1467
error = ret < 0 ? ret : -EFAULT;
1470
handle.page_ptr += page_size;
1471
image_size = image_header->size;
1472
nr_pages = image_header->pages;
1477
/* We have already read one page */
1480
printf("%s: Image size: %lu kilobytes\n", my_name, (unsigned long) image_size / 1024);
1481
real_size = image_size;
1483
handle.swap_needed = image_size;
1485
/* This is necessary in case the image is not compressible */
1486
handle.swap_needed += round_up_page_size(
1487
(handle.swap_needed >> 4) + 67);
1489
if (!enough_swap(&handle)) {
635
1490
fprintf(stderr, "%s: Not enough free swap\n", my_name);
638
error = init_swap_writer(&handle, snapshot_fd, resume_fd,
639
(char *)mem_pool + page_size);
643
header->map_start = handle.cur_swap;
644
header->image_flags = 0;
645
max_block_size = page_size;
1494
if (!preallocate_swap(&handle)) {
1495
fprintf(stderr, "%s: Failed to allocate swap\n", my_name);
1499
/* Shift handle.cur_offset for the first call to next_swap_page() */
1500
handle.cur_offset -= page_size;
1502
header->pages = nr_pages;
1504
header->map_start = handle.extents_spc;
646
1506
if (compute_checksum)
647
header->image_flags |= IMAGE_CHECKSUM;
1507
header->flags |= IMAGE_CHECKSUM;
650
header->image_flags |= IMAGE_COMPRESSED;
652
* The formula below follows from the worst-case expansion
653
* calculation for LZO1. sizeof(short) is due to the size field
654
* in 'struct buf_block'.
656
max_block_size += (page_size >> 4) + 67 + sizeof(short);
1510
header->flags |= IMAGE_COMPRESSED;
659
1512
#ifdef CONFIG_ENCRYPT
660
1513
if (!do_encrypt)
661
1514
goto Save_image;
664
error = gcry_cipher_setkey(cipher_handle, key_data->key,
1517
error = gcry_cipher_setkey(cipher_handle, key_data.key,
669
error = gcry_cipher_setiv(cipher_handle, key_data->ivec,
1522
error = gcry_cipher_setiv(cipher_handle, key_data.ivec,
674
header->image_flags |= IMAGE_ENCRYPTED | IMAGE_USE_RSA;
675
memcpy(&header->rsa, &key_data->rsa, sizeof(struct RSA_data));
676
memcpy(&header->key, &key_data->encrypted_key,
1527
header->flags |= IMAGE_ENCRYPTED | IMAGE_USE_RSA;
1528
memcpy(&header->rsa, &key_data.rsa, sizeof(struct RSA_data));
1529
memcpy(&header->key, &key_data.encrypted_key,
677
1530
sizeof(struct encrypted_key));
682
encrypt_init(key_data->key, key_data->ivec, password);
1535
encrypt_init(key_data.key, key_data.ivec, password);
683
1536
splash.progress(20);
684
1537
get_random_salt(header->salt, CIPHER_BLOCK);
685
1538
for (j = 0; j < CIPHER_BLOCK; j++)
686
key_data->ivec[j] ^= header->salt[j];
1539
key_data.ivec[j] ^= header->salt[j];
688
error = gcry_cipher_setkey(cipher_handle, key_data->key,
1541
error = gcry_cipher_setkey(cipher_handle, key_data.key,
691
error = gcry_cipher_setiv(cipher_handle, key_data->ivec,
1544
error = gcry_cipher_setiv(cipher_handle, key_data.ivec,
694
header->image_flags |= IMAGE_ENCRYPTED;
1547
header->flags |= IMAGE_ENCRYPTED;
698
1551
fprintf(stderr,"%s: libgcrypt error: %s\n", my_name,
699
1552
gcry_strerror(error));
705
1558
gettimeofday(&begin, NULL);
707
error = save_image(&handle, header->pages - 1);
712
* NOTICE: This needs to go after save_image(), because the user may
713
* modify the behavior.
715
if (shutdown_method == SHUTDOWN_METHOD_PLATFORM)
716
header->image_flags |= PLATFORM_SUSPEND;
718
error = flush_swap_writer(&handle);
1560
error = save_image(&handle, nr_pages);
720
1562
struct timeval end;
722
1564
fsync(resume_fd);
724
if (compute_checksum)
1566
header->image_data_size = handle.written_data;
1567
real_size = handle.written_data;
1570
* NOTICE: This needs to go after save_image(), because the
1571
* user may modify the behavior.
1573
if (shutdown_method == SHUTDOWN_METHOD_PLATFORM)
1574
header->flags |= PLATFORM_SUSPEND;
1576
if (compute_checksum || verify_image)
725
1577
md5_finish_ctx(&handle.ctx, header->checksum);
727
1579
gettimeofday(&end, NULL);
728
1580
timersub(&end, &begin, &end);
729
1581
header->writeout_time = end.tv_usec / 1000000.0 + end.tv_sec;
731
error = write_area(resume_fd, header, start, page_size);
1583
header->resume_pause = resume_pause;
1585
error = write_page(resume_fd, header, start);
1590
free_swap_writer(&handle);
1592
if (!error && (verify_image || test_mode)) {
1595
printf("%s: Image verification\n", my_name);
1596
error = read_or_verify(snapshot_fd, resume_fd, header, start,
1597
verify_image, test_mode);
1599
printf(error ? "%s: Image verification failed\n" :
1600
"%s: Image verified successfully\n",
1602
splash.progress(100);
735
1606
if (do_compress) {
736
double delta = header->size - compr_diff;
738
1607
printf("%s: Compression ratio %4.2lf\n", my_name,
739
delta / header->size);
1608
real_size / image_size);
742
1611
error = mark_swap(resume_fd, start);
756
1625
static int reset_signature(int fd)
758
1627
int ret, error = 0;
759
1628
unsigned int size = sizeof(struct swsusp_header);
760
unsigned int shift = (resume_offset + 1) * page_size - size;
1629
off64_t shift = ((off64_t)resume_offset + 1) * page_size - size;
762
if (lseek(fd, shift, SEEK_SET) != shift)
1631
if (lseek64(fd, shift, SEEK_SET) != shift)
765
1634
memset(&swsusp_header, 0, size);