1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/kernel/power/swap.c
4	*
5	* This file provides functions for reading the suspend image from
6	* and writing it to a swap partition.
7	*
8	* Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
9	* Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
10	* Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com>
11	*/
12
13	#define pr_fmt(fmt) "PM: " fmt
14
15	#include <crypto/acompress.h>
16	#include <linux/module.h>
17	#include <linux/file.h>
18	#include <linux/delay.h>
19	#include <linux/bitops.h>
20	#include <linux/device.h>
21	#include <linux/bio.h>
22	#include <linux/blkdev.h>
23	#include <linux/swap.h>
24	#include <linux/swapops.h>
25	#include <linux/pm.h>
26	#include <linux/slab.h>
27	#include <linux/vmalloc.h>
28	#include <linux/cpumask.h>
29	#include <linux/atomic.h>
30	#include <linux/kthread.h>
31	#include <linux/crc32.h>
32	#include <linux/ktime.h>
33
34	#include "power.h"
35
36	#define HIBERNATE_SIG "S1SUSPEND"
37
38	u32 swsusp_hardware_signature;
39
40	/*
41	* When reading an {un,}compressed image, we may restore pages in place,
42	* in which case some architectures need these pages cleaning before they
43	* can be executed. We don't know which pages these may be, so clean the lot.
44	*/
45	static bool clean_pages_on_read;
46	static bool clean_pages_on_decompress;
47
48	/*
49	* The swap map is a data structure used for keeping track of each page
50	* written to a swap partition. It consists of many swap_map_page
51	* structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
52	* These structures are stored on the swap and linked together with the
53	* help of the .next_swap member.
54	*
55	* The swap map is created during suspend. The swap map pages are
56	* allocated and populated one at a time, so we only need one memory
57	* page to set up the entire structure.
58	*
59	* During resume we pick up all swap_map_page structures into a list.
60	*/
61
62	#define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
63
64	/*
65	* Number of free pages that are not high.
66	*/
67	static inline unsigned long low_free_pages(void)
68	{
69	return nr_free_pages() - nr_free_highpages();
70	}
71
72	/*
73	* Number of pages required to be kept free while writing the image. Always
74	* half of all available low pages before the writing starts.
75	*/
76	static inline unsigned long reqd_free_pages(void)
77	{
78	return low_free_pages() / 2;
79	}
80
81	struct swap_map_page {
82	sector_t entries[MAP_PAGE_ENTRIES];
83	sector_t next_swap;
84	};
85
86	struct swap_map_page_list {
87	struct swap_map_page *map;
88	struct swap_map_page_list *next;
89	};
90
91	/*
92	* The swap_map_handle structure is used for handling swap in
93	* a file-alike way
94	*/
95
96	struct swap_map_handle {
97	struct swap_map_page *cur;
98	struct swap_map_page_list *maps;
99	sector_t cur_swap;
100	sector_t first_sector;
101	unsigned int k;
102	unsigned long reqd_free_pages;
103	u32 crc32;
104	};
105
106	struct swsusp_header {
107	char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
108	sizeof(u32) - sizeof(u32)];
109	u32 hw_sig;
110	u32 crc32;
111	sector_t image;
112	unsigned int flags; /* Flags to pass to the "boot" kernel */
113	char orig_sig[10];
114	char sig[10];
115	} __packed;
116
117	static struct swsusp_header *swsusp_header;
118
119	/*
120	* The following functions are used for tracing the allocated
121	* swap pages, so that they can be freed in case of an error.
122	*/
123
124	struct swsusp_extent {
125	struct rb_node node;
126	unsigned long start;
127	unsigned long end;
128	};
129
130	static struct rb_root swsusp_extents = RB_ROOT;
131
132	static int swsusp_extents_insert(unsigned long swap_offset)
133	{
134	struct rb_node **new = &(swsusp_extents.rb_node);
135	struct rb_node *parent = NULL;
136	struct swsusp_extent *ext;
137
138	/* Figure out where to put the new node */
139	while (*new) {
140	ext = rb_entry(*new, struct swsusp_extent, node);
141	parent = *new;
142	if (swap_offset < ext->start) {
143	/* Try to merge */
144	if (swap_offset == ext->start - 1) {
145	ext->start--;
146	return 0;
147	}
148	new = &((*new)->rb_left);
149	} else if (swap_offset > ext->end) {
150	/* Try to merge */
151	if (swap_offset == ext->end + 1) {
152	ext->end++;
153	return 0;
154	}
155	new = &((*new)->rb_right);
156	} else {
157	/* It already is in the tree */
158	return -EINVAL;
159	}
160	}
161	/* Add the new node and rebalance the tree. */
162	ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
163	if (!ext)
164	return -ENOMEM;
165
166	ext->start = swap_offset;
167	ext->end = swap_offset;
168	rb_link_node(node: &ext->node, parent, rb_link: new);
169	rb_insert_color(&ext->node, &swsusp_extents);
170	return 0;
171	}
172
173	/*
174	* alloc_swapdev_block - allocate a swap page and register that it has
175	* been allocated, so that it can be freed in case of an error.
176	*/
177
178	sector_t alloc_swapdev_block(int swap)
179	{
180	unsigned long offset;
181
182	offset = swp_offset(entry: get_swap_page_of_type(swap));
183	if (offset) {
184	if (swsusp_extents_insert(swap_offset: offset))
185	swap_free(entry: swp_entry(type: swap, offset));
186	else
187	return swapdev_block(swap, offset);
188	}
189	return 0;
190	}
191
192	/*
193	* free_all_swap_pages - free swap pages allocated for saving image data.
194	* It also frees the extents used to register which swap entries had been
195	* allocated.
196	*/
197
198	void free_all_swap_pages(int swap)
199	{
200	struct rb_node *node;
201
202	while ((node = swsusp_extents.rb_node)) {
203	struct swsusp_extent *ext;
204
205	ext = rb_entry(node, struct swsusp_extent, node);
206	rb_erase(node, &swsusp_extents);
207	swap_free_nr(entry: swp_entry(type: swap, offset: ext->start),
208	nr_pages: ext->end - ext->start + 1);
209
210	kfree(objp: ext);
211	}
212	}
213
214	int swsusp_swap_in_use(void)
215	{
216	return (swsusp_extents.rb_node != NULL);
217	}
218
219	/*
220	* General things
221	*/
222
223	static unsigned short root_swap = 0xffff;
224	static struct file *hib_resume_bdev_file;
225
226	struct hib_bio_batch {
227	atomic_t count;
228	wait_queue_head_t wait;
229	blk_status_t error;
230	struct blk_plug plug;
231	};
232
233	static void hib_init_batch(struct hib_bio_batch *hb)
234	{
235	atomic_set(v: &hb->count, i: 0);
236	init_waitqueue_head(&hb->wait);
237	hb->error = BLK_STS_OK;
238	blk_start_plug(&hb->plug);
239	}
240
241	static void hib_finish_batch(struct hib_bio_batch *hb)
242	{
243	blk_finish_plug(&hb->plug);
244	}
245
246	static void hib_end_io(struct bio *bio)
247	{
248	struct hib_bio_batch *hb = bio->bi_private;
249	struct page *page = bio_first_page_all(bio);
250
251	if (bio->bi_status) {
252	pr_alert("Read-error on swap-device (%u:%u:%Lu)\n",
253	MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
254	(unsigned long long)bio->bi_iter.bi_sector);
255	}
256
257	if (bio_data_dir(bio) == WRITE)
258	put_page(page);
259	else if (clean_pages_on_read)
260	flush_icache_range(start: (unsigned long)page_address(page),
261	end: (unsigned long)page_address(page) + PAGE_SIZE);
262
263	if (bio->bi_status && !hb->error)
264	hb->error = bio->bi_status;
265	if (atomic_dec_and_test(v: &hb->count))
266	wake_up(&hb->wait);
267
268	bio_put(bio);
269	}
270
271	static int hib_submit_io_sync(blk_opf_t opf, pgoff_t page_off, void *addr)
272	{
273	return bdev_rw_virt(bdev: file_bdev(bdev_file: hib_resume_bdev_file),
274	sector: page_off * (PAGE_SIZE >> 9), data: addr, PAGE_SIZE, op: opf);
275	}
276
277	static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr,
278	struct hib_bio_batch *hb)
279	{
280	struct bio *bio;
281
282	bio = bio_alloc(bdev: file_bdev(bdev_file: hib_resume_bdev_file), nr_vecs: 1, opf,
283	GFP_NOIO | __GFP_HIGH);
284	bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
285	bio_add_virt_nofail(bio, vaddr: addr, PAGE_SIZE);
286	bio->bi_end_io = hib_end_io;
287	bio->bi_private = hb;
288	atomic_inc(v: &hb->count);
289	submit_bio(bio);
290	return 0;
291	}
292
293	static int hib_wait_io(struct hib_bio_batch *hb)
294	{
295	/*
296	* We are relying on the behavior of blk_plug that a thread with
297	* a plug will flush the plug list before sleeping.
298	*/
299	wait_event(hb->wait, atomic_read(&hb->count) == 0);
300	return blk_status_to_errno(status: hb->error);
301	}
302
303	/*
304	* Saving part
305	*/
306	static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
307	{
308	int error;
309
310	hib_submit_io_sync(opf: REQ_OP_READ, page_off: swsusp_resume_block, addr: swsusp_header);
311	if (!memcmp(p: "SWAP-SPACE",q: swsusp_header->sig, size: 10) ||
312	!memcmp(p: "SWAPSPACE2",q: swsusp_header->sig, size: 10)) {
313	memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
314	memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
315	swsusp_header->image = handle->first_sector;
316	if (swsusp_hardware_signature) {
317	swsusp_header->hw_sig = swsusp_hardware_signature;
318	flags |= SF_HW_SIG;
319	}
320	swsusp_header->flags = flags;
321	if (flags & SF_CRC32_MODE)
322	swsusp_header->crc32 = handle->crc32;
323	error = hib_submit_io_sync(opf: REQ_OP_WRITE | REQ_SYNC,
324	page_off: swsusp_resume_block, addr: swsusp_header);
325	} else {
326	pr_err("Swap header not found!\n");
327	error = -ENODEV;
328	}
329	return error;
330	}
331
332	/*
333	* Hold the swsusp_header flag. This is used in software_resume() in
334	* 'kernel/power/hibernate' to check if the image is compressed and query
335	* for the compression algorithm support(if so).
336	*/
337	unsigned int swsusp_header_flags;
338
339	/**
340	* swsusp_swap_check - check if the resume device is a swap device
341	* and get its index (if so)
342	*
343	* This is called before saving image
344	*/
345	static int swsusp_swap_check(void)
346	{
347	int res;
348
349	if (swsusp_resume_device)
350	res = swap_type_of(device: swsusp_resume_device, offset: swsusp_resume_block);
351	else
352	res = find_first_swap(device: &swsusp_resume_device);
353	if (res < 0)
354	return res;
355	root_swap = res;
356
357	hib_resume_bdev_file = bdev_file_open_by_dev(dev: swsusp_resume_device,
358	BLK_OPEN_WRITE, NULL, NULL);
359	if (IS_ERR(ptr: hib_resume_bdev_file))
360	return PTR_ERR(ptr: hib_resume_bdev_file);
361
362	return 0;
363	}
364
365	/**
366	* write_page - Write one page to given swap location.
367	* @buf: Address we're writing.
368	* @offset: Offset of the swap page we're writing to.
369	* @hb: bio completion batch
370	*/
371
372	static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
373	{
374	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;
375	void *src;
376	int ret;
377
378	if (!offset)
379	return -ENOSPC;
380
381	if (!hb)
382	goto sync_io;
383
384	src = (void *)__get_free_page(gfp);
385	if (!src) {
386	ret = hib_wait_io(hb); /* Free pages */
387	if (ret)
388	return ret;
389	src = (void *)__get_free_page(gfp);
390	if (WARN_ON_ONCE(!src))
391	goto sync_io;
392	}
393
394	copy_page(to: src, from: buf);
395	return hib_submit_io_async(opf: REQ_OP_WRITE | REQ_SYNC, page_off: offset, addr: src, hb);
396	sync_io:
397	return hib_submit_io_sync(opf: REQ_OP_WRITE | REQ_SYNC, page_off: offset, addr: buf);
398	}
399
400	static void release_swap_writer(struct swap_map_handle *handle)
401	{
402	if (handle->cur)
403	free_page((unsigned long)handle->cur);
404	handle->cur = NULL;
405	}
406
407	static int get_swap_writer(struct swap_map_handle *handle)
408	{
409	int ret;
410
411	ret = swsusp_swap_check();
412	if (ret) {
413	if (ret != -ENOSPC)
414	pr_err("Cannot find swap device, try swapon -a\n");
415	return ret;
416	}
417	handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
418	if (!handle->cur) {
419	ret = -ENOMEM;
420	goto err_close;
421	}
422	handle->cur_swap = alloc_swapdev_block(swap: root_swap);
423	if (!handle->cur_swap) {
424	ret = -ENOSPC;
425	goto err_rel;
426	}
427	handle->k = 0;
428	handle->reqd_free_pages = reqd_free_pages();
429	handle->first_sector = handle->cur_swap;
430	return 0;
431	err_rel:
432	release_swap_writer(handle);
433	err_close:
434	swsusp_close();
435	return ret;
436	}
437
438	static int swap_write_page(struct swap_map_handle *handle, void *buf,
439	struct hib_bio_batch *hb)
440	{
441	int error;
442	sector_t offset;
443
444	if (!handle->cur)
445	return -EINVAL;
446	offset = alloc_swapdev_block(swap: root_swap);
447	error = write_page(buf, offset, hb);
448	if (error)
449	return error;
450	handle->cur->entries[handle->k++] = offset;
451	if (handle->k >= MAP_PAGE_ENTRIES) {
452	offset = alloc_swapdev_block(swap: root_swap);
453	if (!offset)
454	return -ENOSPC;
455	handle->cur->next_swap = offset;
456	error = write_page(buf: handle->cur, offset: handle->cur_swap, hb);
457	if (error)
458	goto out;
459	clear_page(page: handle->cur);
460	handle->cur_swap = offset;
461	handle->k = 0;
462
463	if (hb && low_free_pages() <= handle->reqd_free_pages) {
464	error = hib_wait_io(hb);
465	if (error)
466	goto out;
467	/*
468	* Recalculate the number of required free pages, to
469	* make sure we never take more than half.
470	*/
471	handle->reqd_free_pages = reqd_free_pages();
472	}
473	}
474	out:
475	return error;
476	}
477
478	static int flush_swap_writer(struct swap_map_handle *handle)
479	{
480	if (handle->cur && handle->cur_swap)
481	return write_page(buf: handle->cur, offset: handle->cur_swap, NULL);
482	else
483	return -EINVAL;
484	}
485
486	static int swap_writer_finish(struct swap_map_handle *handle,
487	unsigned int flags, int error)
488	{
489	if (!error) {
490	pr_info("S");
491	error = mark_swapfiles(handle, flags);
492	pr_cont("|\n");
493	flush_swap_writer(handle);
494	}
495
496	if (error)
497	free_all_swap_pages(swap: root_swap);
498	release_swap_writer(handle);
499	swsusp_close();
500
501	return error;
502	}
503
504	/*
505	* Bytes we need for compressed data in worst case. We assume(limitation)
506	* this is the worst of all the compression algorithms.
507	*/
508	#define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2)
509
510	/* We need to remember how much compressed data we need to read. */
511	#define CMP_HEADER sizeof(size_t)
512
513	/* Number of pages/bytes we'll compress at one time. */
514	#define UNC_PAGES 32
515	#define UNC_SIZE (UNC_PAGES * PAGE_SIZE)
516
517	/* Number of pages we need for compressed data (worst case). */
518	#define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \
519	CMP_HEADER, PAGE_SIZE)
520	#define CMP_SIZE (CMP_PAGES * PAGE_SIZE)
521
522	/* Maximum number of threads for compression/decompression. */
523	#define CMP_THREADS 3
524
525	/* Minimum/maximum number of pages for read buffering. */
526	#define CMP_MIN_RD_PAGES 1024
527	#define CMP_MAX_RD_PAGES 8192
528
529	/**
530	* save_image - save the suspend image data
531	*/
532
533	static int save_image(struct swap_map_handle *handle,
534	struct snapshot_handle *snapshot,
535	unsigned int nr_to_write)
536	{
537	unsigned int m;
538	int ret;
539	int nr_pages;
540	int err2;
541	struct hib_bio_batch hb;
542	ktime_t start;
543	ktime_t stop;
544
545	hib_init_batch(hb: &hb);
546
547	pr_info("Saving image data pages (%u pages)...\n",
548	nr_to_write);
549	m = nr_to_write / 10;
550	if (!m)
551	m = 1;
552	nr_pages = 0;
553	start = ktime_get();
554	while (1) {
555	ret = snapshot_read_next(handle: snapshot);
556	if (ret <= 0)
557	break;
558	ret = swap_write_page(handle, data_of(*snapshot), hb: &hb);
559	if (ret)
560	break;
561	if (!(nr_pages % m))
562	pr_info("Image saving progress: %3d%%\n",
563	nr_pages / m * 10);
564	nr_pages++;
565	}
566	err2 = hib_wait_io(hb: &hb);
567	hib_finish_batch(hb: &hb);
568	stop = ktime_get();
569	if (!ret)
570	ret = err2;
571	if (!ret)
572	pr_info("Image saving done\n");
573	swsusp_show_speed(start, stop, nr_to_write, "Wrote");
574	return ret;
575	}
576
577	/*
578	* Structure used for CRC32.
579	*/
580	struct crc_data {
581	struct task_struct *thr; /* thread */
582	atomic_t ready; /* ready to start flag */
583	atomic_t stop; /* ready to stop flag */
584	unsigned run_threads; /* nr current threads */
585	wait_queue_head_t go; /* start crc update */
586	wait_queue_head_t done; /* crc update done */
587	u32 *crc32; /* points to handle's crc32 */
588	size_t *unc_len[CMP_THREADS]; /* uncompressed lengths */
589	unsigned char *unc[CMP_THREADS]; /* uncompressed data */
590	};
591
592	/*
593	* CRC32 update function that runs in its own thread.
594	*/
595	static int crc32_threadfn(void *data)
596	{
597	struct crc_data *d = data;
598	unsigned i;
599
600	while (1) {
601	wait_event(d->go, atomic_read_acquire(&d->ready) ||
602	kthread_should_stop());
603	if (kthread_should_stop()) {
604	d->thr = NULL;
605	atomic_set_release(v: &d->stop, i: 1);
606	wake_up(&d->done);
607	break;
608	}
609	atomic_set(v: &d->ready, i: 0);
610
611	for (i = 0; i < d->run_threads; i++)
612	*d->crc32 = crc32_le(crc: *d->crc32,
613	p: d->unc[i], len: *d->unc_len[i]);
614	atomic_set_release(v: &d->stop, i: 1);
615	wake_up(&d->done);
616	}
617	return 0;
618	}
619	/*
620	* Structure used for data compression.
621	*/
622	struct cmp_data {
623	struct task_struct *thr; /* thread */
624	struct crypto_acomp *cc; /* crypto compressor */
625	struct acomp_req *cr; /* crypto request */
626	atomic_t ready; /* ready to start flag */
627	atomic_t stop; /* ready to stop flag */
628	int ret; /* return code */
629	wait_queue_head_t go; /* start compression */
630	wait_queue_head_t done; /* compression done */
631	size_t unc_len; /* uncompressed length */
632	size_t cmp_len; /* compressed length */
633	unsigned char unc[UNC_SIZE]; /* uncompressed buffer */
634	unsigned char cmp[CMP_SIZE]; /* compressed buffer */
635	};
636
637	/* Indicates the image size after compression */
638	static atomic_t compressed_size = ATOMIC_INIT(0);
639
640	/*
641	* Compression function that runs in its own thread.
642	*/
643	static int compress_threadfn(void *data)
644	{
645	struct cmp_data *d = data;
646
647	while (1) {
648	wait_event(d->go, atomic_read_acquire(&d->ready) ||
649	kthread_should_stop());
650	if (kthread_should_stop()) {
651	d->thr = NULL;
652	d->ret = -1;
653	atomic_set_release(v: &d->stop, i: 1);
654	wake_up(&d->done);
655	break;
656	}
657	atomic_set(v: &d->ready, i: 0);
658
659	acomp_request_set_callback(req: d->cr, CRYPTO_TFM_REQ_MAY_SLEEP,
660	NULL, NULL);
661	acomp_request_set_src_nondma(req: d->cr, src: d->unc, slen: d->unc_len);
662	acomp_request_set_dst_nondma(req: d->cr, dst: d->cmp + CMP_HEADER,
663	CMP_SIZE - CMP_HEADER);
664	d->ret = crypto_acomp_compress(req: d->cr);
665	d->cmp_len = d->cr->dlen;
666
667	atomic_set(v: &compressed_size, i: atomic_read(v: &compressed_size) + d->cmp_len);
668	atomic_set_release(v: &d->stop, i: 1);
669	wake_up(&d->done);
670	}
671	return 0;
672	}
673
674	/**
675	* save_compressed_image - Save the suspend image data after compression.
676	* @handle: Swap map handle to use for saving the image.
677	* @snapshot: Image to read data from.
678	* @nr_to_write: Number of pages to save.
679	*/
680	static int save_compressed_image(struct swap_map_handle *handle,
681	struct snapshot_handle *snapshot,
682	unsigned int nr_to_write)
683	{
684	unsigned int m;
685	int ret = 0;
686	int nr_pages;
687	int err2;
688	struct hib_bio_batch hb;
689	ktime_t start;
690	ktime_t stop;
691	size_t off;
692	unsigned thr, run_threads, nr_threads;
693	unsigned char *page = NULL;
694	struct cmp_data *data = NULL;
695	struct crc_data *crc = NULL;
696
697	hib_init_batch(hb: &hb);
698
699	atomic_set(v: &compressed_size, i: 0);
700
701	/*
702	* We'll limit the number of threads for compression to limit memory
703	* footprint.
704	*/
705	nr_threads = num_online_cpus() - 1;
706	nr_threads = clamp_val(nr_threads, 1, CMP_THREADS);
707
708	page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH);
709	if (!page) {
710	pr_err("Failed to allocate %s page\n", hib_comp_algo);
711	ret = -ENOMEM;
712	goto out_clean;
713	}
714
715	data = vzalloc(array_size(nr_threads, sizeof(*data)));
716	if (!data) {
717	pr_err("Failed to allocate %s data\n", hib_comp_algo);
718	ret = -ENOMEM;
719	goto out_clean;
720	}
721
722	crc = kzalloc(sizeof(*crc), GFP_KERNEL);
723	if (!crc) {
724	pr_err("Failed to allocate crc\n");
725	ret = -ENOMEM;
726	goto out_clean;
727	}
728
729	/*
730	* Start the compression threads.
731	*/
732	for (thr = 0; thr < nr_threads; thr++) {
733	init_waitqueue_head(&data[thr].go);
734	init_waitqueue_head(&data[thr].done);
735
736	data[thr].cc = crypto_alloc_acomp(alg_name: hib_comp_algo, type: 0, CRYPTO_ALG_ASYNC);
737	if (IS_ERR_OR_NULL(ptr: data[thr].cc)) {
738	pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
739	ret = -EFAULT;
740	goto out_clean;
741	}
742
743	data[thr].cr = acomp_request_alloc(data[thr].cc);
744	if (!data[thr].cr) {
745	pr_err("Could not allocate comp request\n");
746	ret = -ENOMEM;
747	goto out_clean;
748	}
749
750	data[thr].thr = kthread_run(compress_threadfn,
751	&data[thr],
752	"image_compress/%u", thr);
753	if (IS_ERR(ptr: data[thr].thr)) {
754	data[thr].thr = NULL;
755	pr_err("Cannot start compression threads\n");
756	ret = -ENOMEM;
757	goto out_clean;
758	}
759	}
760
761	/*
762	* Start the CRC32 thread.
763	*/
764	init_waitqueue_head(&crc->go);
765	init_waitqueue_head(&crc->done);
766
767	handle->crc32 = 0;
768	crc->crc32 = &handle->crc32;
769	for (thr = 0; thr < nr_threads; thr++) {
770	crc->unc[thr] = data[thr].unc;
771	crc->unc_len[thr] = &data[thr].unc_len;
772	}
773
774	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
775	if (IS_ERR(ptr: crc->thr)) {
776	crc->thr = NULL;
777	pr_err("Cannot start CRC32 thread\n");
778	ret = -ENOMEM;
779	goto out_clean;
780	}
781
782	/*
783	* Adjust the number of required free pages after all allocations have
784	* been done. We don't want to run out of pages when writing.
785	*/
786	handle->reqd_free_pages = reqd_free_pages();
787
788	pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo);
789	pr_info("Compressing and saving image data (%u pages)...\n",
790	nr_to_write);
791	m = nr_to_write / 10;
792	if (!m)
793	m = 1;
794	nr_pages = 0;
795	start = ktime_get();
796	for (;;) {
797	for (thr = 0; thr < nr_threads; thr++) {
798	for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) {
799	ret = snapshot_read_next(handle: snapshot);
800	if (ret < 0)
801	goto out_finish;
802
803	if (!ret)
804	break;
805
806	memcpy(data[thr].unc + off,
807	data_of(*snapshot), PAGE_SIZE);
808
809	if (!(nr_pages % m))
810	pr_info("Image saving progress: %3d%%\n",
811	nr_pages / m * 10);
812	nr_pages++;
813	}
814	if (!off)
815	break;
816
817	data[thr].unc_len = off;
818
819	atomic_set_release(v: &data[thr].ready, i: 1);
820	wake_up(&data[thr].go);
821	}
822
823	if (!thr)
824	break;
825
826	crc->run_threads = thr;
827	atomic_set_release(v: &crc->ready, i: 1);
828	wake_up(&crc->go);
829
830	for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
831	wait_event(data[thr].done,
832	atomic_read_acquire(&data[thr].stop));
833	atomic_set(v: &data[thr].stop, i: 0);
834
835	ret = data[thr].ret;
836
837	if (ret < 0) {
838	pr_err("%s compression failed\n", hib_comp_algo);
839	goto out_finish;
840	}
841
842	if (unlikely(!data[thr].cmp_len ||
843	data[thr].cmp_len >
844	bytes_worst_compress(data[thr].unc_len))) {
845	pr_err("Invalid %s compressed length\n", hib_comp_algo);
846	ret = -1;
847	goto out_finish;
848	}
849
850	*(size_t *)data[thr].cmp = data[thr].cmp_len;
851
852	/*
853	* Given we are writing one page at a time to disk, we
854	* copy that much from the buffer, although the last
855	* bit will likely be smaller than full page. This is
856	* OK - we saved the length of the compressed data, so
857	* any garbage at the end will be discarded when we
858	* read it.
859	*/
860	for (off = 0;
861	off < CMP_HEADER + data[thr].cmp_len;
862	off += PAGE_SIZE) {
863	memcpy(page, data[thr].cmp + off, PAGE_SIZE);
864
865	ret = swap_write_page(handle, buf: page, hb: &hb);
866	if (ret)
867	goto out_finish;
868	}
869	}
870
871	wait_event(crc->done, atomic_read_acquire(&crc->stop));
872	atomic_set(v: &crc->stop, i: 0);
873	}
874
875	out_finish:
876	err2 = hib_wait_io(hb: &hb);
877	stop = ktime_get();
878	if (!ret)
879	ret = err2;
880	if (!ret)
881	pr_info("Image saving done\n");
882	swsusp_show_speed(start, stop, nr_to_write, "Wrote");
883	pr_info("Image size after compression: %d kbytes\n",
884	(atomic_read(&compressed_size) / 1024));
885
886	out_clean:
887	hib_finish_batch(hb: &hb);
888	if (crc) {
889	if (crc->thr)
890	kthread_stop(k: crc->thr);
891	kfree(objp: crc);
892	}
893	if (data) {
894	for (thr = 0; thr < nr_threads; thr++) {
895	if (data[thr].thr)
896	kthread_stop(k: data[thr].thr);
897	acomp_request_free(req: data[thr].cr);
898	crypto_free_acomp(tfm: data[thr].cc);
899	}
900	vfree(addr: data);
901	}
902	if (page) free_page((unsigned long)page);
903
904	return ret;
905	}
906
907	/**
908	* enough_swap - Make sure we have enough swap to save the image.
909	*
910	* Returns TRUE or FALSE after checking the total amount of swap
911	* space available from the resume partition.
912	*/
913
914	static int enough_swap(unsigned int nr_pages)
915	{
916	unsigned int free_swap = count_swap_pages(root_swap, 1);
917	unsigned int required;
918
919	pr_debug("Free swap pages: %u\n", free_swap);
920
921	required = PAGES_FOR_IO + nr_pages;
922	return free_swap > required;
923	}
924
925	/**
926	* swsusp_write - Write entire image and metadata.
927	* @flags: flags to pass to the "boot" kernel in the image header
928	*
929	* It is important _NOT_ to umount filesystems at this point. We want
930	* them synced (in case something goes wrong) but we DO not want to mark
931	* filesystem clean: it is not. (And it does not matter, if we resume
932	* correctly, we'll mark system clean, anyway.)
933	*/
934
935	int swsusp_write(unsigned int flags)
936	{
937	struct swap_map_handle handle;
938	struct snapshot_handle snapshot;
939	struct swsusp_info *header;
940	unsigned long pages;
941	int error;
942
943	pages = snapshot_get_image_size();
944	error = get_swap_writer(handle: &handle);
945	if (error) {
946	pr_err("Cannot get swap writer\n");
947	return error;
948	}
949	if (flags & SF_NOCOMPRESS_MODE) {
950	if (!enough_swap(nr_pages: pages)) {
951	pr_err("Not enough free swap\n");
952	error = -ENOSPC;
953	goto out_finish;
954	}
955	}
956	memset(&snapshot, 0, sizeof(struct snapshot_handle));
957	error = snapshot_read_next(handle: &snapshot);
958	if (error < (int)PAGE_SIZE) {
959	if (error >= 0)
960	error = -EFAULT;
961
962	goto out_finish;
963	}
964	header = (struct swsusp_info *)data_of(snapshot);
965	error = swap_write_page(handle: &handle, buf: header, NULL);
966	if (!error) {
967	error = (flags & SF_NOCOMPRESS_MODE) ?
968	save_image(handle: &handle, snapshot: &snapshot, nr_to_write: pages - 1) :
969	save_compressed_image(handle: &handle, snapshot: &snapshot, nr_to_write: pages - 1);
970	}
971	out_finish:
972	error = swap_writer_finish(handle: &handle, flags, error);
973	return error;
974	}
975
976	/*
977	* The following functions allow us to read data using a swap map
978	* in a file-like way.
979	*/
980
981	static void release_swap_reader(struct swap_map_handle *handle)
982	{
983	struct swap_map_page_list *tmp;
984
985	while (handle->maps) {
986	if (handle->maps->map)
987	free_page((unsigned long)handle->maps->map);
988	tmp = handle->maps;
989	handle->maps = handle->maps->next;
990	kfree(objp: tmp);
991	}
992	handle->cur = NULL;
993	}
994
995	static int get_swap_reader(struct swap_map_handle *handle,
996	unsigned int *flags_p)
997	{
998	int error;
999	struct swap_map_page_list *tmp, *last;
1000	sector_t offset;
1001
1002	*flags_p = swsusp_header->flags;
1003
1004	if (!swsusp_header->image) /* how can this happen? */
1005	return -EINVAL;
1006
1007	handle->cur = NULL;
1008	last = handle->maps = NULL;
1009	offset = swsusp_header->image;
1010	while (offset) {
1011	tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL);
1012	if (!tmp) {
1013	release_swap_reader(handle);
1014	return -ENOMEM;
1015	}
1016	if (!handle->maps)
1017	handle->maps = tmp;
1018	if (last)
1019	last->next = tmp;
1020	last = tmp;
1021
1022	tmp->map = (struct swap_map_page *)
1023	__get_free_page(GFP_NOIO | __GFP_HIGH);
1024	if (!tmp->map) {
1025	release_swap_reader(handle);
1026	return -ENOMEM;
1027	}
1028
1029	error = hib_submit_io_sync(opf: REQ_OP_READ, page_off: offset, addr: tmp->map);
1030	if (error) {
1031	release_swap_reader(handle);
1032	return error;
1033	}
1034	offset = tmp->map->next_swap;
1035	}
1036	handle->k = 0;
1037	handle->cur = handle->maps->map;
1038	return 0;
1039	}
1040
1041	static int swap_read_page(struct swap_map_handle *handle, void *buf,
1042	struct hib_bio_batch *hb)
1043	{
1044	sector_t offset;
1045	int error;
1046	struct swap_map_page_list *tmp;
1047
1048	if (!handle->cur)
1049	return -EINVAL;
1050	offset = handle->cur->entries[handle->k];
1051	if (!offset)
1052	return -EFAULT;
1053	if (hb)
1054	error = hib_submit_io_async(opf: REQ_OP_READ, page_off: offset, addr: buf, hb);
1055	else
1056	error = hib_submit_io_sync(opf: REQ_OP_READ, page_off: offset, addr: buf);
1057	if (error)
1058	return error;
1059	if (++handle->k >= MAP_PAGE_ENTRIES) {
1060	handle->k = 0;
1061	free_page((unsigned long)handle->maps->map);
1062	tmp = handle->maps;
1063	handle->maps = handle->maps->next;
1064	kfree(objp: tmp);
1065	if (!handle->maps)
1066	release_swap_reader(handle);
1067	else
1068	handle->cur = handle->maps->map;
1069	}
1070	return error;
1071	}
1072
1073	static int swap_reader_finish(struct swap_map_handle *handle)
1074	{
1075	release_swap_reader(handle);
1076
1077	return 0;
1078	}
1079
1080	/**
1081	* load_image - load the image using the swap map handle
1082	* @handle and the snapshot handle @snapshot
1083	* (assume there are @nr_pages pages to load)
1084	*/
1085
1086	static int load_image(struct swap_map_handle *handle,
1087	struct snapshot_handle *snapshot,
1088	unsigned int nr_to_read)
1089	{
1090	unsigned int m;
1091	int ret = 0;
1092	ktime_t start;
1093	ktime_t stop;
1094	struct hib_bio_batch hb;
1095	int err2;
1096	unsigned nr_pages;
1097
1098	hib_init_batch(hb: &hb);
1099
1100	clean_pages_on_read = true;
1101	pr_info("Loading image data pages (%u pages)...\n", nr_to_read);
1102	m = nr_to_read / 10;
1103	if (!m)
1104	m = 1;
1105	nr_pages = 0;
1106	start = ktime_get();
1107	for ( ; ; ) {
1108	ret = snapshot_write_next(handle: snapshot);
1109	if (ret <= 0)
1110	break;
1111	ret = swap_read_page(handle, data_of(*snapshot), hb: &hb);
1112	if (ret)
1113	break;
1114	if (snapshot->sync_read)
1115	ret = hib_wait_io(hb: &hb);
1116	if (ret)
1117	break;
1118	if (!(nr_pages % m))
1119	pr_info("Image loading progress: %3d%%\n",
1120	nr_pages / m * 10);
1121	nr_pages++;
1122	}
1123	err2 = hib_wait_io(hb: &hb);
1124	hib_finish_batch(hb: &hb);
1125	stop = ktime_get();
1126	if (!ret)
1127	ret = err2;
1128	if (!ret) {
1129	pr_info("Image loading done\n");
1130	ret = snapshot_write_finalize(handle: snapshot);
1131	if (!ret && !snapshot_image_loaded(handle: snapshot))
1132	ret = -ENODATA;
1133	}
1134	swsusp_show_speed(start, stop, nr_to_read, "Read");
1135	return ret;
1136	}
1137
1138	/*
1139	* Structure used for data decompression.
1140	*/
1141	struct dec_data {
1142	struct task_struct *thr; /* thread */
1143	struct crypto_acomp *cc; /* crypto compressor */
1144	struct acomp_req *cr; /* crypto request */
1145	atomic_t ready; /* ready to start flag */
1146	atomic_t stop; /* ready to stop flag */
1147	int ret; /* return code */
1148	wait_queue_head_t go; /* start decompression */
1149	wait_queue_head_t done; /* decompression done */
1150	size_t unc_len; /* uncompressed length */
1151	size_t cmp_len; /* compressed length */
1152	unsigned char unc[UNC_SIZE]; /* uncompressed buffer */
1153	unsigned char cmp[CMP_SIZE]; /* compressed buffer */
1154	};
1155
1156	/*
1157	* Decompression function that runs in its own thread.
1158	*/
1159	static int decompress_threadfn(void *data)
1160	{
1161	struct dec_data *d = data;
1162
1163	while (1) {
1164	wait_event(d->go, atomic_read_acquire(&d->ready) ||
1165	kthread_should_stop());
1166	if (kthread_should_stop()) {
1167	d->thr = NULL;
1168	d->ret = -1;
1169	atomic_set_release(v: &d->stop, i: 1);
1170	wake_up(&d->done);
1171	break;
1172	}
1173	atomic_set(v: &d->ready, i: 0);
1174
1175	acomp_request_set_callback(req: d->cr, CRYPTO_TFM_REQ_MAY_SLEEP,
1176	NULL, NULL);
1177	acomp_request_set_src_nondma(req: d->cr, src: d->cmp + CMP_HEADER,
1178	slen: d->cmp_len);
1179	acomp_request_set_dst_nondma(req: d->cr, dst: d->unc, UNC_SIZE);
1180	d->ret = crypto_acomp_decompress(req: d->cr);
1181	d->unc_len = d->cr->dlen;
1182
1183	if (clean_pages_on_decompress)
1184	flush_icache_range(start: (unsigned long)d->unc,
1185	end: (unsigned long)d->unc + d->unc_len);
1186
1187	atomic_set_release(v: &d->stop, i: 1);
1188	wake_up(&d->done);
1189	}
1190	return 0;
1191	}
1192
1193	/**
1194	* load_compressed_image - Load compressed image data and decompress it.
1195	* @handle: Swap map handle to use for loading data.
1196	* @snapshot: Image to copy uncompressed data into.
1197	* @nr_to_read: Number of pages to load.
1198	*/
1199	static int load_compressed_image(struct swap_map_handle *handle,
1200	struct snapshot_handle *snapshot,
1201	unsigned int nr_to_read)
1202	{
1203	unsigned int m;
1204	int ret = 0;
1205	int eof = 0;
1206	struct hib_bio_batch hb;
1207	ktime_t start;
1208	ktime_t stop;
1209	unsigned nr_pages;
1210	size_t off;
1211	unsigned i, thr, run_threads, nr_threads;
1212	unsigned ring = 0, pg = 0, ring_size = 0,
1213	have = 0, want, need, asked = 0;
1214	unsigned long read_pages = 0;
1215	unsigned char **page = NULL;
1216	struct dec_data *data = NULL;
1217	struct crc_data *crc = NULL;
1218
1219	hib_init_batch(hb: &hb);
1220
1221	/*
1222	* We'll limit the number of threads for decompression to limit memory
1223	* footprint.
1224	*/
1225	nr_threads = num_online_cpus() - 1;
1226	nr_threads = clamp_val(nr_threads, 1, CMP_THREADS);
1227
1228	page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page)));
1229	if (!page) {
1230	pr_err("Failed to allocate %s page\n", hib_comp_algo);
1231	ret = -ENOMEM;
1232	goto out_clean;
1233	}
1234
1235	data = vzalloc(array_size(nr_threads, sizeof(*data)));
1236	if (!data) {
1237	pr_err("Failed to allocate %s data\n", hib_comp_algo);
1238	ret = -ENOMEM;
1239	goto out_clean;
1240	}
1241
1242	crc = kzalloc(sizeof(*crc), GFP_KERNEL);
1243	if (!crc) {
1244	pr_err("Failed to allocate crc\n");
1245	ret = -ENOMEM;
1246	goto out_clean;
1247	}
1248
1249	clean_pages_on_decompress = true;
1250
1251	/*
1252	* Start the decompression threads.
1253	*/
1254	for (thr = 0; thr < nr_threads; thr++) {
1255	init_waitqueue_head(&data[thr].go);
1256	init_waitqueue_head(&data[thr].done);
1257
1258	data[thr].cc = crypto_alloc_acomp(alg_name: hib_comp_algo, type: 0, CRYPTO_ALG_ASYNC);
1259	if (IS_ERR_OR_NULL(ptr: data[thr].cc)) {
1260	pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
1261	ret = -EFAULT;
1262	goto out_clean;
1263	}
1264
1265	data[thr].cr = acomp_request_alloc(data[thr].cc);
1266	if (!data[thr].cr) {
1267	pr_err("Could not allocate comp request\n");
1268	ret = -ENOMEM;
1269	goto out_clean;
1270	}
1271
1272	data[thr].thr = kthread_run(decompress_threadfn,
1273	&data[thr],
1274	"image_decompress/%u", thr);
1275	if (IS_ERR(ptr: data[thr].thr)) {
1276	data[thr].thr = NULL;
1277	pr_err("Cannot start decompression threads\n");
1278	ret = -ENOMEM;
1279	goto out_clean;
1280	}
1281	}
1282
1283	/*
1284	* Start the CRC32 thread.
1285	*/
1286	init_waitqueue_head(&crc->go);
1287	init_waitqueue_head(&crc->done);
1288
1289	handle->crc32 = 0;
1290	crc->crc32 = &handle->crc32;
1291	for (thr = 0; thr < nr_threads; thr++) {
1292	crc->unc[thr] = data[thr].unc;
1293	crc->unc_len[thr] = &data[thr].unc_len;
1294	}
1295
1296	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
1297	if (IS_ERR(ptr: crc->thr)) {
1298	crc->thr = NULL;
1299	pr_err("Cannot start CRC32 thread\n");
1300	ret = -ENOMEM;
1301	goto out_clean;
1302	}
1303
1304	/*
1305	* Set the number of pages for read buffering.
1306	* This is complete guesswork, because we'll only know the real
1307	* picture once prepare_image() is called, which is much later on
1308	* during the image load phase. We'll assume the worst case and
1309	* say that none of the image pages are from high memory.
1310	*/
1311	if (low_free_pages() > snapshot_get_image_size())
1312	read_pages = (low_free_pages() - snapshot_get_image_size()) / 2;
1313	read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES);
1314
1315	for (i = 0; i < read_pages; i++) {
1316	page[i] = (void *)__get_free_page(i < CMP_PAGES ?
1317	GFP_NOIO | __GFP_HIGH :
1318	GFP_NOIO | __GFP_NOWARN |
1319	__GFP_NORETRY);
1320
1321	if (!page[i]) {
1322	if (i < CMP_PAGES) {
1323	ring_size = i;
1324	pr_err("Failed to allocate %s pages\n", hib_comp_algo);
1325	ret = -ENOMEM;
1326	goto out_clean;
1327	} else {
1328	break;
1329	}
1330	}
1331	}
1332	want = ring_size = i;
1333
1334	pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo);
1335	pr_info("Loading and decompressing image data (%u pages)...\n",
1336	nr_to_read);
1337	m = nr_to_read / 10;
1338	if (!m)
1339	m = 1;
1340	nr_pages = 0;
1341	start = ktime_get();
1342
1343	ret = snapshot_write_next(handle: snapshot);
1344	if (ret <= 0)
1345	goto out_finish;
1346
1347	for(;;) {
1348	for (i = 0; !eof && i < want; i++) {
1349	ret = swap_read_page(handle, buf: page[ring], hb: &hb);
1350	if (ret) {
1351	/*
1352	* On real read error, finish. On end of data,
1353	* set EOF flag and just exit the read loop.
1354	*/
1355	if (handle->cur &&
1356	handle->cur->entries[handle->k]) {
1357	goto out_finish;
1358	} else {
1359	eof = 1;
1360	break;
1361	}
1362	}
1363	if (++ring >= ring_size)
1364	ring = 0;
1365	}
1366	asked += i;
1367	want -= i;
1368
1369	/*
1370	* We are out of data, wait for some more.
1371	*/
1372	if (!have) {
1373	if (!asked)
1374	break;
1375
1376	ret = hib_wait_io(hb: &hb);
1377	if (ret)
1378	goto out_finish;
1379	have += asked;
1380	asked = 0;
1381	if (eof)
1382	eof = 2;
1383	}
1384
1385	if (crc->run_threads) {
1386	wait_event(crc->done, atomic_read_acquire(&crc->stop));
1387	atomic_set(v: &crc->stop, i: 0);
1388	crc->run_threads = 0;
1389	}
1390
1391	for (thr = 0; have && thr < nr_threads; thr++) {
1392	data[thr].cmp_len = *(size_t *)page[pg];
1393	if (unlikely(!data[thr].cmp_len ||
1394	data[thr].cmp_len >
1395	bytes_worst_compress(UNC_SIZE))) {
1396	pr_err("Invalid %s compressed length\n", hib_comp_algo);
1397	ret = -1;
1398	goto out_finish;
1399	}
1400
1401	need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER,
1402	PAGE_SIZE);
1403	if (need > have) {
1404	if (eof > 1) {
1405	ret = -1;
1406	goto out_finish;
1407	}
1408	break;
1409	}
1410
1411	for (off = 0;
1412	off < CMP_HEADER + data[thr].cmp_len;
1413	off += PAGE_SIZE) {
1414	memcpy(data[thr].cmp + off,
1415	page[pg], PAGE_SIZE);
1416	have--;
1417	want++;
1418	if (++pg >= ring_size)
1419	pg = 0;
1420	}
1421
1422	atomic_set_release(v: &data[thr].ready, i: 1);
1423	wake_up(&data[thr].go);
1424	}
1425
1426	/*
1427	* Wait for more data while we are decompressing.
1428	*/
1429	if (have < CMP_PAGES && asked) {
1430	ret = hib_wait_io(hb: &hb);
1431	if (ret)
1432	goto out_finish;
1433	have += asked;
1434	asked = 0;
1435	if (eof)
1436	eof = 2;
1437	}
1438
1439	for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
1440	wait_event(data[thr].done,
1441	atomic_read_acquire(&data[thr].stop));
1442	atomic_set(v: &data[thr].stop, i: 0);
1443
1444	ret = data[thr].ret;
1445
1446	if (ret < 0) {
1447	pr_err("%s decompression failed\n", hib_comp_algo);
1448	goto out_finish;
1449	}
1450
1451	if (unlikely(!data[thr].unc_len ||
1452	data[thr].unc_len > UNC_SIZE ||
1453	data[thr].unc_len & (PAGE_SIZE - 1))) {
1454	pr_err("Invalid %s uncompressed length\n", hib_comp_algo);
1455	ret = -1;
1456	goto out_finish;
1457	}
1458
1459	for (off = 0;
1460	off < data[thr].unc_len; off += PAGE_SIZE) {
1461	memcpy(data_of(*snapshot),
1462	data[thr].unc + off, PAGE_SIZE);
1463
1464	if (!(nr_pages % m))
1465	pr_info("Image loading progress: %3d%%\n",
1466	nr_pages / m * 10);
1467	nr_pages++;
1468
1469	ret = snapshot_write_next(handle: snapshot);
1470	if (ret <= 0) {
1471	crc->run_threads = thr + 1;
1472	atomic_set_release(v: &crc->ready, i: 1);
1473	wake_up(&crc->go);
1474	goto out_finish;
1475	}
1476	}
1477	}
1478
1479	crc->run_threads = thr;
1480	atomic_set_release(v: &crc->ready, i: 1);
1481	wake_up(&crc->go);
1482	}
1483
1484	out_finish:
1485	if (crc->run_threads) {
1486	wait_event(crc->done, atomic_read_acquire(&crc->stop));
1487	atomic_set(v: &crc->stop, i: 0);
1488	}
1489	stop = ktime_get();
1490	if (!ret) {
1491	pr_info("Image loading done\n");
1492	ret = snapshot_write_finalize(handle: snapshot);
1493	if (!ret && !snapshot_image_loaded(handle: snapshot))
1494	ret = -ENODATA;
1495	if (!ret) {
1496	if (swsusp_header->flags & SF_CRC32_MODE) {
1497	if(handle->crc32 != swsusp_header->crc32) {
1498	pr_err("Invalid image CRC32!\n");
1499	ret = -ENODATA;
1500	}
1501	}
1502	}
1503	}
1504	swsusp_show_speed(start, stop, nr_to_read, "Read");
1505	out_clean:
1506	hib_finish_batch(hb: &hb);
1507	for (i = 0; i < ring_size; i++)
1508	free_page((unsigned long)page[i]);
1509	if (crc) {
1510	if (crc->thr)
1511	kthread_stop(k: crc->thr);
1512	kfree(objp: crc);
1513	}
1514	if (data) {
1515	for (thr = 0; thr < nr_threads; thr++) {
1516	if (data[thr].thr)
1517	kthread_stop(k: data[thr].thr);
1518	acomp_request_free(req: data[thr].cr);
1519	crypto_free_acomp(tfm: data[thr].cc);
1520	}
1521	vfree(addr: data);
1522	}
1523	vfree(addr: page);
1524
1525	return ret;
1526	}
1527
1528	/**
1529	* swsusp_read - read the hibernation image.
1530	* @flags_p: flags passed by the "frozen" kernel in the image header should
1531	* be written into this memory location
1532	*/
1533
1534	int swsusp_read(unsigned int *flags_p)
1535	{
1536	int error;
1537	struct swap_map_handle handle;
1538	struct snapshot_handle snapshot;
1539	struct swsusp_info *header;
1540
1541	memset(&snapshot, 0, sizeof(struct snapshot_handle));
1542	error = snapshot_write_next(handle: &snapshot);
1543	if (error < (int)PAGE_SIZE)
1544	return error < 0 ? error : -EFAULT;
1545	header = (struct swsusp_info *)data_of(snapshot);
1546	error = get_swap_reader(handle: &handle, flags_p);
1547	if (error)
1548	goto end;
1549	if (!error)
1550	error = swap_read_page(handle: &handle, buf: header, NULL);
1551	if (!error) {
1552	error = (*flags_p & SF_NOCOMPRESS_MODE) ?
1553	load_image(handle: &handle, snapshot: &snapshot, nr_to_read: header->pages - 1) :
1554	load_compressed_image(handle: &handle, snapshot: &snapshot, nr_to_read: header->pages - 1);
1555	}
1556	swap_reader_finish(handle: &handle);
1557	end:
1558	if (!error)
1559	pr_debug("Image successfully loaded\n");
1560	else
1561	pr_debug("Error %d resuming\n", error);
1562	return error;
1563	}
1564
1565	static void *swsusp_holder;
1566
1567	/**
1568	* swsusp_check - Open the resume device and check for the swsusp signature.
1569	* @exclusive: Open the resume device exclusively.
1570	*/
1571
1572	int swsusp_check(bool exclusive)
1573	{
1574	void *holder = exclusive ? &swsusp_holder : NULL;
1575	int error;
1576
1577	hib_resume_bdev_file = bdev_file_open_by_dev(dev: swsusp_resume_device,
1578	BLK_OPEN_READ, holder, NULL);
1579	if (!IS_ERR(ptr: hib_resume_bdev_file)) {
1580	clear_page(page: swsusp_header);
1581	error = hib_submit_io_sync(opf: REQ_OP_READ, page_off: swsusp_resume_block,
1582	addr: swsusp_header);
1583	if (error)
1584	goto put;
1585
1586	if (!memcmp(HIBERNATE_SIG, q: swsusp_header->sig, size: 10)) {
1587	memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
1588	swsusp_header_flags = swsusp_header->flags;
1589	/* Reset swap signature now */
1590	error = hib_submit_io_sync(opf: REQ_OP_WRITE | REQ_SYNC,
1591	page_off: swsusp_resume_block,
1592	addr: swsusp_header);
1593	} else {
1594	error = -EINVAL;
1595	}
1596	if (!error && swsusp_header->flags & SF_HW_SIG &&
1597	swsusp_header->hw_sig != swsusp_hardware_signature) {
1598	pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n",
1599	swsusp_header->hw_sig, swsusp_hardware_signature);
1600	error = -EINVAL;
1601	}
1602
1603	put:
1604	if (error)
1605	bdev_fput(bdev_file: hib_resume_bdev_file);
1606	else
1607	pr_debug("Image signature found, resuming\n");
1608	} else {
1609	error = PTR_ERR(ptr: hib_resume_bdev_file);
1610	}
1611
1612	if (error)
1613	pr_debug("Image not found (code %d)\n", error);
1614
1615	return error;
1616	}
1617
1618	/**
1619	* swsusp_close - close resume device.
1620	*/
1621
1622	void swsusp_close(void)
1623	{
1624	if (IS_ERR(ptr: hib_resume_bdev_file)) {
1625	pr_debug("Image device not initialised\n");
1626	return;
1627	}
1628
1629	fput(hib_resume_bdev_file);
1630	}
1631
1632	/**
1633	* swsusp_unmark - Unmark swsusp signature in the resume device
1634	*/
1635
1636	#ifdef CONFIG_SUSPEND
1637	int swsusp_unmark(void)
1638	{
1639	int error;
1640
1641	hib_submit_io_sync(opf: REQ_OP_READ, page_off: swsusp_resume_block, addr: swsusp_header);
1642	if (!memcmp(HIBERNATE_SIG,q: swsusp_header->sig, size: 10)) {
1643	memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
1644	error = hib_submit_io_sync(opf: REQ_OP_WRITE | REQ_SYNC,
1645	page_off: swsusp_resume_block,
1646	addr: swsusp_header);
1647	} else {
1648	pr_err("Cannot find swsusp signature!\n");
1649	error = -ENODEV;
1650	}
1651
1652	/*
1653	* We just returned from suspend, we don't need the image any more.
1654	*/
1655	free_all_swap_pages(swap: root_swap);
1656
1657	return error;
1658	}
1659	#endif
1660
1661	static int __init swsusp_header_init(void)
1662	{
1663	swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
1664	if (!swsusp_header)
1665	panic(fmt: "Could not allocate memory for swsusp_header\n");
1666	return 0;
1667	}
1668
1669	core_initcall(swsusp_header_init);
1670