1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2003 Jana Saout <jana@saout.de>
4	* Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
5	* Copyright (C) 2006-2020 Red Hat, Inc. All rights reserved.
6	* Copyright (C) 2013-2020 Milan Broz <gmazyland@gmail.com>
7	*
8	* This file is released under the GPL.
9	*/
10
11	#include <linux/completion.h>
12	#include <linux/err.h>
13	#include <linux/module.h>
14	#include <linux/init.h>
15	#include <linux/kernel.h>
16	#include <linux/key.h>
17	#include <linux/bio.h>
18	#include <linux/blkdev.h>
19	#include <linux/blk-integrity.h>
20	#include <linux/crc32.h>
21	#include <linux/mempool.h>
22	#include <linux/slab.h>
23	#include <linux/crypto.h>
24	#include <linux/workqueue.h>
25	#include <linux/kthread.h>
26	#include <linux/backing-dev.h>
27	#include <linux/atomic.h>
28	#include <linux/scatterlist.h>
29	#include <linux/rbtree.h>
30	#include <linux/ctype.h>
31	#include <asm/page.h>
32	#include <linux/unaligned.h>
33	#include <crypto/hash.h>
34	#include <crypto/md5.h>
35	#include <crypto/skcipher.h>
36	#include <crypto/aead.h>
37	#include <crypto/authenc.h>
38	#include <crypto/utils.h>
39	#include <linux/rtnetlink.h> /* for struct rtattr and RTA macros only */
40	#include <linux/key-type.h>
41	#include <keys/user-type.h>
42	#include <keys/encrypted-type.h>
43	#include <keys/trusted-type.h>
44
45	#include <linux/device-mapper.h>
46
47	#include "dm-audit.h"
48
49	#define DM_MSG_PREFIX "crypt"
50
51	static DEFINE_IDA(workqueue_ida);
52
53	/*
54	* context holding the current state of a multi-part conversion
55	*/
56	struct convert_context {
57	struct completion restart;
58	struct bio *bio_in;
59	struct bvec_iter iter_in;
60	struct bio *bio_out;
61	struct bvec_iter iter_out;
62	atomic_t cc_pending;
63	unsigned int tag_offset;
64	u64 cc_sector;
65	union {
66	struct skcipher_request *req;
67	struct aead_request *req_aead;
68	} r;
69	bool aead_recheck;
70	bool aead_failed;
71
72	};
73
74	/*
75	* per bio private data
76	*/
77	struct dm_crypt_io {
78	struct crypt_config *cc;
79	struct bio *base_bio;
80	u8 *integrity_metadata;
81	bool integrity_metadata_from_pool:1;
82
83	struct work_struct work;
84
85	struct convert_context ctx;
86
87	atomic_t io_pending;
88	blk_status_t error;
89	sector_t sector;
90
91	struct bvec_iter saved_bi_iter;
92
93	struct rb_node rb_node;
94	} CRYPTO_MINALIGN_ATTR;
95
96	struct dm_crypt_request {
97	struct convert_context *ctx;
98	struct scatterlist sg_in[4];
99	struct scatterlist sg_out[4];
100	u64 iv_sector;
101	};
102
103	struct crypt_config;
104
105	struct crypt_iv_operations {
106	int (*ctr)(struct crypt_config *cc, struct dm_target *ti,
107	const char *opts);
108	void (*dtr)(struct crypt_config *cc);
109	int (*init)(struct crypt_config *cc);
110	int (*wipe)(struct crypt_config *cc);
111	int (*generator)(struct crypt_config *cc, u8 *iv,
112	struct dm_crypt_request *dmreq);
113	int (*post)(struct crypt_config *cc, u8 *iv,
114	struct dm_crypt_request *dmreq);
115	};
116
117	struct iv_benbi_private {
118	int shift;
119	};
120
121	#define LMK_SEED_SIZE 64 /* hash + 0 */
122	struct iv_lmk_private {
123	struct crypto_shash *hash_tfm;
124	u8 *seed;
125	};
126
127	#define TCW_WHITENING_SIZE 16
128	struct iv_tcw_private {
129	u8 *iv_seed;
130	u8 *whitening;
131	};
132
133	#define ELEPHANT_MAX_KEY_SIZE 32
134	struct iv_elephant_private {
135	struct crypto_skcipher *tfm;
136	};
137
138	/*
139	* Crypt: maps a linear range of a block device
140	* and encrypts / decrypts at the same time.
141	*/
142	enum flags { DM_CRYPT_SUSPENDED, DM_CRYPT_KEY_VALID,
143	DM_CRYPT_SAME_CPU, DM_CRYPT_HIGH_PRIORITY,
144	DM_CRYPT_NO_OFFLOAD, DM_CRYPT_NO_READ_WORKQUEUE,
145	DM_CRYPT_NO_WRITE_WORKQUEUE, DM_CRYPT_WRITE_INLINE };
146
147	enum cipher_flags {
148	CRYPT_MODE_INTEGRITY_AEAD, /* Use authenticated mode for cipher */
149	CRYPT_IV_LARGE_SECTORS, /* Calculate IV from sector_size, not 512B sectors */
150	CRYPT_ENCRYPT_PREPROCESS, /* Must preprocess data for encryption (elephant) */
151	CRYPT_KEY_MAC_SIZE_SET, /* The integrity_key_size option was used */
152	};
153
154	/*
155	* The fields in here must be read only after initialization.
156	*/
157	struct crypt_config {
158	struct dm_dev *dev;
159	sector_t start;
160
161	struct percpu_counter n_allocated_pages;
162
163	struct workqueue_struct *io_queue;
164	struct workqueue_struct *crypt_queue;
165
166	spinlock_t write_thread_lock;
167	struct task_struct *write_thread;
168	struct rb_root write_tree;
169
170	char *cipher_string;
171	char *cipher_auth;
172	char *key_string;
173
174	const struct crypt_iv_operations *iv_gen_ops;
175	union {
176	struct iv_benbi_private benbi;
177	struct iv_lmk_private lmk;
178	struct iv_tcw_private tcw;
179	struct iv_elephant_private elephant;
180	} iv_gen_private;
181	u64 iv_offset;
182	unsigned int iv_size;
183	unsigned short sector_size;
184	unsigned char sector_shift;
185
186	union {
187	struct crypto_skcipher **tfms;
188	struct crypto_aead **tfms_aead;
189	} cipher_tfm;
190	unsigned int tfms_count;
191	int workqueue_id;
192	unsigned long cipher_flags;
193
194	/*
195	* Layout of each crypto request:
196	*
197	* struct skcipher_request
198	* context
199	* padding
200	* struct dm_crypt_request
201	* padding
202	* IV
203	*
204	* The padding is added so that dm_crypt_request and the IV are
205	* correctly aligned.
206	*/
207	unsigned int dmreq_start;
208
209	unsigned int per_bio_data_size;
210
211	unsigned long flags;
212	unsigned int key_size;
213	unsigned int key_parts; /* independent parts in key buffer */
214	unsigned int key_extra_size; /* additional keys length */
215	unsigned int key_mac_size; /* MAC key size for authenc(...) */
216
217	unsigned int integrity_tag_size;
218	unsigned int integrity_iv_size;
219	unsigned int used_tag_size;
220	unsigned int tuple_size;
221
222	/*
223	* pool for per bio private data, crypto requests,
224	* encryption requeusts/buffer pages and integrity tags
225	*/
226	unsigned int tag_pool_max_sectors;
227	mempool_t tag_pool;
228	mempool_t req_pool;
229	mempool_t page_pool;
230
231	struct bio_set bs;
232	struct mutex bio_alloc_lock;
233
234	u8 *authenc_key; /* space for keys in authenc() format (if used) */
235	u8 key[] __counted_by(key_size);
236	};
237
238	#define MIN_IOS 64
239	#define MAX_TAG_SIZE 480
240	#define POOL_ENTRY_SIZE 512
241
242	static DEFINE_SPINLOCK(dm_crypt_clients_lock);
243	static unsigned int dm_crypt_clients_n;
244	static volatile unsigned long dm_crypt_pages_per_client;
245	#define DM_CRYPT_MEMORY_PERCENT 2
246	#define DM_CRYPT_MIN_PAGES_PER_CLIENT (BIO_MAX_VECS * 16)
247	#define DM_CRYPT_DEFAULT_MAX_READ_SIZE 131072
248	#define DM_CRYPT_DEFAULT_MAX_WRITE_SIZE 131072
249
250	static unsigned int max_read_size = 0;
251	module_param(max_read_size, uint, 0644);
252	MODULE_PARM_DESC(max_read_size, "Maximum size of a read request");
253	static unsigned int max_write_size = 0;
254	module_param(max_write_size, uint, 0644);
255	MODULE_PARM_DESC(max_write_size, "Maximum size of a write request");
256	static unsigned get_max_request_size(struct crypt_config *cc, bool wrt)
257	{
258	unsigned val, sector_align;
259	val = !wrt ? READ_ONCE(max_read_size) : READ_ONCE(max_write_size);
260	if (likely(!val))
261	val = !wrt ? DM_CRYPT_DEFAULT_MAX_READ_SIZE : DM_CRYPT_DEFAULT_MAX_WRITE_SIZE;
262	if (wrt || cc->used_tag_size) {
263	if (unlikely(val > BIO_MAX_VECS << PAGE_SHIFT))
264	val = BIO_MAX_VECS << PAGE_SHIFT;
265	}
266	sector_align = max(bdev_logical_block_size(cc->dev->bdev), (unsigned)cc->sector_size);
267	val = round_down(val, sector_align);
268	if (unlikely(!val))
269	val = sector_align;
270	return val >> SECTOR_SHIFT;
271	}
272
273	static void crypt_endio(struct bio *clone);
274	static void kcryptd_queue_crypt(struct dm_crypt_io *io);
275	static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
276	struct scatterlist *sg);
277
278	static bool crypt_integrity_aead(struct crypt_config *cc);
279
280	/*
281	* Use this to access cipher attributes that are independent of the key.
282	*/
283	static struct crypto_skcipher *any_tfm(struct crypt_config *cc)
284	{
285	return cc->cipher_tfm.tfms[0];
286	}
287
288	static struct crypto_aead *any_tfm_aead(struct crypt_config *cc)
289	{
290	return cc->cipher_tfm.tfms_aead[0];
291	}
292
293	/*
294	* Different IV generation algorithms:
295	*
296	* plain: the initial vector is the 32-bit little-endian version of the sector
297	* number, padded with zeros if necessary.
298	*
299	* plain64: the initial vector is the 64-bit little-endian version of the sector
300	* number, padded with zeros if necessary.
301	*
302	* plain64be: the initial vector is the 64-bit big-endian version of the sector
303	* number, padded with zeros if necessary.
304	*
305	* essiv: "encrypted sector|salt initial vector", the sector number is
306	* encrypted with the bulk cipher using a salt as key. The salt
307	* should be derived from the bulk cipher's key via hashing.
308	*
309	* benbi: the 64-bit "big-endian 'narrow block'-count", starting at 1
310	* (needed for LRW-32-AES and possible other narrow block modes)
311	*
312	* null: the initial vector is always zero. Provides compatibility with
313	* obsolete loop_fish2 devices. Do not use for new devices.
314	*
315	* lmk: Compatible implementation of the block chaining mode used
316	* by the Loop-AES block device encryption system
317	* designed by Jari Ruusu. See http://loop-aes.sourceforge.net/
318	* It operates on full 512 byte sectors and uses CBC
319	* with an IV derived from the sector number, the data and
320	* optionally extra IV seed.
321	* This means that after decryption the first block
322	* of sector must be tweaked according to decrypted data.
323	* Loop-AES can use three encryption schemes:
324	* version 1: is plain aes-cbc mode
325	* version 2: uses 64 multikey scheme with lmk IV generator
326	* version 3: the same as version 2 with additional IV seed
327	* (it uses 65 keys, last key is used as IV seed)
328	*
329	* tcw: Compatible implementation of the block chaining mode used
330	* by the TrueCrypt device encryption system (prior to version 4.1).
331	* For more info see: https://gitlab.com/cryptsetup/cryptsetup/wikis/TrueCryptOnDiskFormat
332	* It operates on full 512 byte sectors and uses CBC
333	* with an IV derived from initial key and the sector number.
334	* In addition, whitening value is applied on every sector, whitening
335	* is calculated from initial key, sector number and mixed using CRC32.
336	* Note that this encryption scheme is vulnerable to watermarking attacks
337	* and should be used for old compatible containers access only.
338	*
339	* eboiv: Encrypted byte-offset IV (used in Bitlocker in CBC mode)
340	* The IV is encrypted little-endian byte-offset (with the same key
341	* and cipher as the volume).
342	*
343	* elephant: The extended version of eboiv with additional Elephant diffuser
344	* used with Bitlocker CBC mode.
345	* This mode was used in older Windows systems
346	* https://download.microsoft.com/download/0/2/3/0238acaf-d3bf-4a6d-b3d6-0a0be4bbb36e/bitlockercipher200608.pdf
347	*/
348
349	static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv,
350	struct dm_crypt_request *dmreq)
351	{
352	memset(iv, 0, cc->iv_size);
353	*(__le32 *)iv = cpu_to_le32(dmreq->iv_sector & 0xffffffff);
354
355	return 0;
356	}
357
358	static int crypt_iv_plain64_gen(struct crypt_config *cc, u8 *iv,
359	struct dm_crypt_request *dmreq)
360	{
361	memset(iv, 0, cc->iv_size);
362	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
363
364	return 0;
365	}
366
367	static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv,
368	struct dm_crypt_request *dmreq)
369	{
370	memset(iv, 0, cc->iv_size);
371	/* iv_size is at least of size u64; usually it is 16 bytes */
372	*(__be64 *)&iv[cc->iv_size - sizeof(u64)] = cpu_to_be64(dmreq->iv_sector);
373
374	return 0;
375	}
376
377	static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv,
378	struct dm_crypt_request *dmreq)
379	{
380	/*
381	* ESSIV encryption of the IV is now handled by the crypto API,
382	* so just pass the plain sector number here.
383	*/
384	memset(iv, 0, cc->iv_size);
385	*(__le64 *)iv = cpu_to_le64(dmreq->iv_sector);
386
387	return 0;
388	}
389
390	static int crypt_iv_benbi_ctr(struct crypt_config *cc, struct dm_target *ti,
391	const char *opts)
392	{
393	unsigned int bs;
394	int log;
395
396	if (crypt_integrity_aead(cc))
397	bs = crypto_aead_blocksize(tfm: any_tfm_aead(cc));
398	else
399	bs = crypto_skcipher_blocksize(tfm: any_tfm(cc));
400	log = ilog2(bs);
401
402	/*
403	* We need to calculate how far we must shift the sector count
404	* to get the cipher block count, we use this shift in _gen.
405	*/
406	if (1 << log != bs) {
407	ti->error = "cypher blocksize is not a power of 2";
408	return -EINVAL;
409	}
410
411	if (log > 9) {
412	ti->error = "cypher blocksize is > 512";
413	return -EINVAL;
414	}
415
416	cc->iv_gen_private.benbi.shift = 9 - log;
417
418	return 0;
419	}
420
421	static void crypt_iv_benbi_dtr(struct crypt_config *cc)
422	{
423	}
424
425	static int crypt_iv_benbi_gen(struct crypt_config *cc, u8 *iv,
426	struct dm_crypt_request *dmreq)
427	{
428	__be64 val;
429
430	memset(iv, 0, cc->iv_size - sizeof(u64)); /* rest is cleared below */
431
432	val = cpu_to_be64(((u64)dmreq->iv_sector << cc->iv_gen_private.benbi.shift) + 1);
433	put_unaligned(val, (__be64 *)(iv + cc->iv_size - sizeof(u64)));
434
435	return 0;
436	}
437
438	static int crypt_iv_null_gen(struct crypt_config *cc, u8 *iv,
439	struct dm_crypt_request *dmreq)
440	{
441	memset(iv, 0, cc->iv_size);
442
443	return 0;
444	}
445
446	static void crypt_iv_lmk_dtr(struct crypt_config *cc)
447	{
448	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
449
450	if (lmk->hash_tfm && !IS_ERR(ptr: lmk->hash_tfm))
451	crypto_free_shash(tfm: lmk->hash_tfm);
452	lmk->hash_tfm = NULL;
453
454	kfree_sensitive(objp: lmk->seed);
455	lmk->seed = NULL;
456	}
457
458	static int crypt_iv_lmk_ctr(struct crypt_config *cc, struct dm_target *ti,
459	const char *opts)
460	{
461	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
462
463	if (cc->sector_size != (1 << SECTOR_SHIFT)) {
464	ti->error = "Unsupported sector size for LMK";
465	return -EINVAL;
466	}
467
468	lmk->hash_tfm = crypto_alloc_shash(alg_name: "md5", type: 0,
469	CRYPTO_ALG_ALLOCATES_MEMORY);
470	if (IS_ERR(ptr: lmk->hash_tfm)) {
471	ti->error = "Error initializing LMK hash";
472	return PTR_ERR(ptr: lmk->hash_tfm);
473	}
474
475	/* No seed in LMK version 2 */
476	if (cc->key_parts == cc->tfms_count) {
477	lmk->seed = NULL;
478	return 0;
479	}
480
481	lmk->seed = kzalloc(LMK_SEED_SIZE, GFP_KERNEL);
482	if (!lmk->seed) {
483	crypt_iv_lmk_dtr(cc);
484	ti->error = "Error kmallocing seed storage in LMK";
485	return -ENOMEM;
486	}
487
488	return 0;
489	}
490
491	static int crypt_iv_lmk_init(struct crypt_config *cc)
492	{
493	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
494	int subkey_size = cc->key_size / cc->key_parts;
495
496	/* LMK seed is on the position of LMK_KEYS + 1 key */
497	if (lmk->seed)
498	memcpy(lmk->seed, cc->key + (cc->tfms_count * subkey_size),
499	crypto_shash_digestsize(lmk->hash_tfm));
500
501	return 0;
502	}
503
504	static int crypt_iv_lmk_wipe(struct crypt_config *cc)
505	{
506	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
507
508	if (lmk->seed)
509	memset(lmk->seed, 0, LMK_SEED_SIZE);
510
511	return 0;
512	}
513
514	static int crypt_iv_lmk_one(struct crypt_config *cc, u8 *iv,
515	struct dm_crypt_request *dmreq,
516	u8 *data)
517	{
518	struct iv_lmk_private *lmk = &cc->iv_gen_private.lmk;
519	SHASH_DESC_ON_STACK(desc, lmk->hash_tfm);
520	struct md5_state md5state;
521	__le32 buf[4];
522	int i, r;
523
524	desc->tfm = lmk->hash_tfm;
525
526	r = crypto_shash_init(desc);
527	if (r)
528	return r;
529
530	if (lmk->seed) {
531	r = crypto_shash_update(desc, data: lmk->seed, LMK_SEED_SIZE);
532	if (r)
533	return r;
534	}
535
536	/* Sector is always 512B, block size 16, add data of blocks 1-31 */
537	r = crypto_shash_update(desc, data: data + 16, len: 16 * 31);
538	if (r)
539	return r;
540
541	/* Sector is cropped to 56 bits here */
542	buf[0] = cpu_to_le32(dmreq->iv_sector & 0xFFFFFFFF);
543	buf[1] = cpu_to_le32((((u64)dmreq->iv_sector >> 32) & 0x00FFFFFF) | 0x80000000);
544	buf[2] = cpu_to_le32(4024);
545	buf[3] = 0;
546	r = crypto_shash_update(desc, data: (u8 *)buf, len: sizeof(buf));
547	if (r)
548	return r;
549
550	/* No MD5 padding here */
551	r = crypto_shash_export(desc, out: &md5state);
552	if (r)
553	return r;
554
555	for (i = 0; i < MD5_HASH_WORDS; i++)
556	__cpu_to_le32s(&md5state.hash[i]);
557	memcpy(iv, &md5state.hash, cc->iv_size);
558
559	return 0;
560	}
561
562	static int crypt_iv_lmk_gen(struct crypt_config *cc, u8 *iv,
563	struct dm_crypt_request *dmreq)
564	{
565	struct scatterlist *sg;
566	u8 *src;
567	int r = 0;
568
569	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
570	sg = crypt_get_sg_data(cc, sg: dmreq->sg_in);
571	src = kmap_local_page(page: sg_page(sg));
572	r = crypt_iv_lmk_one(cc, iv, dmreq, data: src + sg->offset);
573	kunmap_local(src);
574	} else
575	memset(iv, 0, cc->iv_size);
576
577	return r;
578	}
579
580	static int crypt_iv_lmk_post(struct crypt_config *cc, u8 *iv,
581	struct dm_crypt_request *dmreq)
582	{
583	struct scatterlist *sg;
584	u8 *dst;
585	int r;
586
587	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE)
588	return 0;
589
590	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
591	dst = kmap_local_page(page: sg_page(sg));
592	r = crypt_iv_lmk_one(cc, iv, dmreq, data: dst + sg->offset);
593
594	/* Tweak the first block of plaintext sector */
595	if (!r)
596	crypto_xor(dst: dst + sg->offset, src: iv, size: cc->iv_size);
597
598	kunmap_local(dst);
599	return r;
600	}
601
602	static void crypt_iv_tcw_dtr(struct crypt_config *cc)
603	{
604	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
605
606	kfree_sensitive(objp: tcw->iv_seed);
607	tcw->iv_seed = NULL;
608	kfree_sensitive(objp: tcw->whitening);
609	tcw->whitening = NULL;
610	}
611
612	static int crypt_iv_tcw_ctr(struct crypt_config *cc, struct dm_target *ti,
613	const char *opts)
614	{
615	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
616
617	if (cc->sector_size != (1 << SECTOR_SHIFT)) {
618	ti->error = "Unsupported sector size for TCW";
619	return -EINVAL;
620	}
621
622	if (cc->key_size <= (cc->iv_size + TCW_WHITENING_SIZE)) {
623	ti->error = "Wrong key size for TCW";
624	return -EINVAL;
625	}
626
627	tcw->iv_seed = kzalloc(cc->iv_size, GFP_KERNEL);
628	tcw->whitening = kzalloc(TCW_WHITENING_SIZE, GFP_KERNEL);
629	if (!tcw->iv_seed || !tcw->whitening) {
630	crypt_iv_tcw_dtr(cc);
631	ti->error = "Error allocating seed storage in TCW";
632	return -ENOMEM;
633	}
634
635	return 0;
636	}
637
638	static int crypt_iv_tcw_init(struct crypt_config *cc)
639	{
640	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
641	int key_offset = cc->key_size - cc->iv_size - TCW_WHITENING_SIZE;
642
643	memcpy(tcw->iv_seed, &cc->key[key_offset], cc->iv_size);
644	memcpy(tcw->whitening, &cc->key[key_offset + cc->iv_size],
645	TCW_WHITENING_SIZE);
646
647	return 0;
648	}
649
650	static int crypt_iv_tcw_wipe(struct crypt_config *cc)
651	{
652	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
653
654	memset(tcw->iv_seed, 0, cc->iv_size);
655	memset(tcw->whitening, 0, TCW_WHITENING_SIZE);
656
657	return 0;
658	}
659
660	static void crypt_iv_tcw_whitening(struct crypt_config *cc,
661	struct dm_crypt_request *dmreq, u8 *data)
662	{
663	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
664	__le64 sector = cpu_to_le64(dmreq->iv_sector);
665	u8 buf[TCW_WHITENING_SIZE];
666	int i;
667
668	/* xor whitening with sector number */
669	crypto_xor_cpy(dst: buf, src1: tcw->whitening, src2: (u8 *)&sector, size: 8);
670	crypto_xor_cpy(dst: &buf[8], src1: tcw->whitening + 8, src2: (u8 *)&sector, size: 8);
671
672	/* calculate crc32 for every 32bit part and xor it */
673	for (i = 0; i < 4; i++)
674	put_unaligned_le32(crc32(0, &buf[i * 4], 4), p: &buf[i * 4]);
675	crypto_xor(dst: &buf[0], src: &buf[12], size: 4);
676	crypto_xor(dst: &buf[4], src: &buf[8], size: 4);
677
678	/* apply whitening (8 bytes) to whole sector */
679	for (i = 0; i < ((1 << SECTOR_SHIFT) / 8); i++)
680	crypto_xor(dst: data + i * 8, src: buf, size: 8);
681	memzero_explicit(s: buf, count: sizeof(buf));
682	}
683
684	static int crypt_iv_tcw_gen(struct crypt_config *cc, u8 *iv,
685	struct dm_crypt_request *dmreq)
686	{
687	struct scatterlist *sg;
688	struct iv_tcw_private *tcw = &cc->iv_gen_private.tcw;
689	__le64 sector = cpu_to_le64(dmreq->iv_sector);
690	u8 *src;
691
692	/* Remove whitening from ciphertext */
693	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
694	sg = crypt_get_sg_data(cc, sg: dmreq->sg_in);
695	src = kmap_local_page(page: sg_page(sg));
696	crypt_iv_tcw_whitening(cc, dmreq, data: src + sg->offset);
697	kunmap_local(src);
698	}
699
700	/* Calculate IV */
701	crypto_xor_cpy(dst: iv, src1: tcw->iv_seed, src2: (u8 *)&sector, size: 8);
702	if (cc->iv_size > 8)
703	crypto_xor_cpy(dst: &iv[8], src1: tcw->iv_seed + 8, src2: (u8 *)&sector,
704	size: cc->iv_size - 8);
705
706	return 0;
707	}
708
709	static int crypt_iv_tcw_post(struct crypt_config *cc, u8 *iv,
710	struct dm_crypt_request *dmreq)
711	{
712	struct scatterlist *sg;
713	u8 *dst;
714
715	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
716	return 0;
717
718	/* Apply whitening on ciphertext */
719	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
720	dst = kmap_local_page(page: sg_page(sg));
721	crypt_iv_tcw_whitening(cc, dmreq, data: dst + sg->offset);
722	kunmap_local(dst);
723
724	return 0;
725	}
726
727	static int crypt_iv_random_gen(struct crypt_config *cc, u8 *iv,
728	struct dm_crypt_request *dmreq)
729	{
730	/* Used only for writes, there must be an additional space to store IV */
731	get_random_bytes(buf: iv, len: cc->iv_size);
732	return 0;
733	}
734
735	static int crypt_iv_eboiv_ctr(struct crypt_config *cc, struct dm_target *ti,
736	const char *opts)
737	{
738	if (crypt_integrity_aead(cc)) {
739	ti->error = "AEAD transforms not supported for EBOIV";
740	return -EINVAL;
741	}
742
743	if (crypto_skcipher_blocksize(tfm: any_tfm(cc)) != cc->iv_size) {
744	ti->error = "Block size of EBOIV cipher does not match IV size of block cipher";
745	return -EINVAL;
746	}
747
748	return 0;
749	}
750
751	static int crypt_iv_eboiv_gen(struct crypt_config *cc, u8 *iv,
752	struct dm_crypt_request *dmreq)
753	{
754	struct crypto_skcipher *tfm = any_tfm(cc);
755	struct skcipher_request *req;
756	struct scatterlist src, dst;
757	DECLARE_CRYPTO_WAIT(wait);
758	unsigned int reqsize;
759	int err;
760	u8 *buf;
761
762	reqsize = sizeof(*req) + crypto_skcipher_reqsize(tfm);
763	reqsize = ALIGN(reqsize, __alignof__(__le64));
764
765	req = kmalloc(reqsize + cc->iv_size, GFP_NOIO);
766	if (!req)
767	return -ENOMEM;
768
769	skcipher_request_set_tfm(req, tfm);
770
771	buf = (u8 *)req + reqsize;
772	memset(buf, 0, cc->iv_size);
773	*(__le64 *)buf = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
774
775	sg_init_one(&src, page_address(ZERO_PAGE(0)), cc->iv_size);
776	sg_init_one(&dst, iv, cc->iv_size);
777	skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: cc->iv_size, iv: buf);
778	skcipher_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
779	err = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
780	kfree_sensitive(objp: req);
781
782	return err;
783	}
784
785	static void crypt_iv_elephant_dtr(struct crypt_config *cc)
786	{
787	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
788
789	crypto_free_skcipher(tfm: elephant->tfm);
790	elephant->tfm = NULL;
791	}
792
793	static int crypt_iv_elephant_ctr(struct crypt_config *cc, struct dm_target *ti,
794	const char *opts)
795	{
796	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
797	int r;
798
799	elephant->tfm = crypto_alloc_skcipher(alg_name: "ecb(aes)", type: 0,
800	CRYPTO_ALG_ALLOCATES_MEMORY);
801	if (IS_ERR(ptr: elephant->tfm)) {
802	r = PTR_ERR(ptr: elephant->tfm);
803	elephant->tfm = NULL;
804	return r;
805	}
806
807	r = crypt_iv_eboiv_ctr(cc, ti, NULL);
808	if (r)
809	crypt_iv_elephant_dtr(cc);
810	return r;
811	}
812
813	static void diffuser_disk_to_cpu(u32 *d, size_t n)
814	{
815	#ifndef __LITTLE_ENDIAN
816	int i;
817
818	for (i = 0; i < n; i++)
819	d[i] = le32_to_cpu((__le32)d[i]);
820	#endif
821	}
822
823	static void diffuser_cpu_to_disk(__le32 *d, size_t n)
824	{
825	#ifndef __LITTLE_ENDIAN
826	int i;
827
828	for (i = 0; i < n; i++)
829	d[i] = cpu_to_le32((u32)d[i]);
830	#endif
831	}
832
833	static void diffuser_a_decrypt(u32 *d, size_t n)
834	{
835	int i, i1, i2, i3;
836
837	for (i = 0; i < 5; i++) {
838	i1 = 0;
839	i2 = n - 2;
840	i3 = n - 5;
841
842	while (i1 < (n - 1)) {
843	d[i1] += d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
844	i1++; i2++; i3++;
845
846	if (i3 >= n)
847	i3 -= n;
848
849	d[i1] += d[i2] ^ d[i3];
850	i1++; i2++; i3++;
851
852	if (i2 >= n)
853	i2 -= n;
854
855	d[i1] += d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
856	i1++; i2++; i3++;
857
858	d[i1] += d[i2] ^ d[i3];
859	i1++; i2++; i3++;
860	}
861	}
862	}
863
864	static void diffuser_a_encrypt(u32 *d, size_t n)
865	{
866	int i, i1, i2, i3;
867
868	for (i = 0; i < 5; i++) {
869	i1 = n - 1;
870	i2 = n - 2 - 1;
871	i3 = n - 5 - 1;
872
873	while (i1 > 0) {
874	d[i1] -= d[i2] ^ d[i3];
875	i1--; i2--; i3--;
876
877	d[i1] -= d[i2] ^ (d[i3] << 13 | d[i3] >> 19);
878	i1--; i2--; i3--;
879
880	if (i2 < 0)
881	i2 += n;
882
883	d[i1] -= d[i2] ^ d[i3];
884	i1--; i2--; i3--;
885
886	if (i3 < 0)
887	i3 += n;
888
889	d[i1] -= d[i2] ^ (d[i3] << 9 | d[i3] >> 23);
890	i1--; i2--; i3--;
891	}
892	}
893	}
894
895	static void diffuser_b_decrypt(u32 *d, size_t n)
896	{
897	int i, i1, i2, i3;
898
899	for (i = 0; i < 3; i++) {
900	i1 = 0;
901	i2 = 2;
902	i3 = 5;
903
904	while (i1 < (n - 1)) {
905	d[i1] += d[i2] ^ d[i3];
906	i1++; i2++; i3++;
907
908	d[i1] += d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
909	i1++; i2++; i3++;
910
911	if (i2 >= n)
912	i2 -= n;
913
914	d[i1] += d[i2] ^ d[i3];
915	i1++; i2++; i3++;
916
917	if (i3 >= n)
918	i3 -= n;
919
920	d[i1] += d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
921	i1++; i2++; i3++;
922	}
923	}
924	}
925
926	static void diffuser_b_encrypt(u32 *d, size_t n)
927	{
928	int i, i1, i2, i3;
929
930	for (i = 0; i < 3; i++) {
931	i1 = n - 1;
932	i2 = 2 - 1;
933	i3 = 5 - 1;
934
935	while (i1 > 0) {
936	d[i1] -= d[i2] ^ (d[i3] << 25 | d[i3] >> 7);
937	i1--; i2--; i3--;
938
939	if (i3 < 0)
940	i3 += n;
941
942	d[i1] -= d[i2] ^ d[i3];
943	i1--; i2--; i3--;
944
945	if (i2 < 0)
946	i2 += n;
947
948	d[i1] -= d[i2] ^ (d[i3] << 10 | d[i3] >> 22);
949	i1--; i2--; i3--;
950
951	d[i1] -= d[i2] ^ d[i3];
952	i1--; i2--; i3--;
953	}
954	}
955	}
956
957	static int crypt_iv_elephant(struct crypt_config *cc, struct dm_crypt_request *dmreq)
958	{
959	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
960	u8 *es, *ks, *data, *data2, *data_offset;
961	struct skcipher_request *req;
962	struct scatterlist *sg, *sg2, src, dst;
963	DECLARE_CRYPTO_WAIT(wait);
964	int i, r;
965
966	req = skcipher_request_alloc(elephant->tfm, GFP_NOIO);
967	es = kzalloc(16, GFP_NOIO); /* Key for AES */
968	ks = kzalloc(32, GFP_NOIO); /* Elephant sector key */
969
970	if (!req || !es || !ks) {
971	r = -ENOMEM;
972	goto out;
973	}
974
975	*(__le64 *)es = cpu_to_le64(dmreq->iv_sector * cc->sector_size);
976
977	/* E(Ks, e(s)) */
978	sg_init_one(&src, es, 16);
979	sg_init_one(&dst, ks, 16);
980	skcipher_request_set_crypt(req, src: &src, dst: &dst, cryptlen: 16, NULL);
981	skcipher_request_set_callback(req, flags: 0, compl: crypto_req_done, data: &wait);
982	r = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
983	if (r)
984	goto out;
985
986	/* E(Ks, e'(s)) */
987	es[15] = 0x80;
988	sg_init_one(&dst, &ks[16], 16);
989	r = crypto_wait_req(err: crypto_skcipher_encrypt(req), wait: &wait);
990	if (r)
991	goto out;
992
993	sg = crypt_get_sg_data(cc, sg: dmreq->sg_out);
994	data = kmap_local_page(page: sg_page(sg));
995	data_offset = data + sg->offset;
996
997	/* Cannot modify original bio, copy to sg_out and apply Elephant to it */
998	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
999	sg2 = crypt_get_sg_data(cc, sg: dmreq->sg_in);
1000	data2 = kmap_local_page(page: sg_page(sg: sg2));
1001	memcpy(data_offset, data2 + sg2->offset, cc->sector_size);
1002	kunmap_local(data2);
1003	}
1004
1005	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE) {
1006	diffuser_disk_to_cpu(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1007	diffuser_b_decrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1008	diffuser_a_decrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1009	diffuser_cpu_to_disk(d: (__le32 *)data_offset, n: cc->sector_size / sizeof(u32));
1010	}
1011
1012	for (i = 0; i < (cc->sector_size / 32); i++)
1013	crypto_xor(dst: data_offset + i * 32, src: ks, size: 32);
1014
1015	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1016	diffuser_disk_to_cpu(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1017	diffuser_a_encrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1018	diffuser_b_encrypt(d: (u32 *)data_offset, n: cc->sector_size / sizeof(u32));
1019	diffuser_cpu_to_disk(d: (__le32 *)data_offset, n: cc->sector_size / sizeof(u32));
1020	}
1021
1022	kunmap_local(data);
1023	out:
1024	kfree_sensitive(objp: ks);
1025	kfree_sensitive(objp: es);
1026	skcipher_request_free(req);
1027	return r;
1028	}
1029
1030	static int crypt_iv_elephant_gen(struct crypt_config *cc, u8 *iv,
1031	struct dm_crypt_request *dmreq)
1032	{
1033	int r;
1034
1035	if (bio_data_dir(dmreq->ctx->bio_in) == WRITE) {
1036	r = crypt_iv_elephant(cc, dmreq);
1037	if (r)
1038	return r;
1039	}
1040
1041	return crypt_iv_eboiv_gen(cc, iv, dmreq);
1042	}
1043
1044	static int crypt_iv_elephant_post(struct crypt_config *cc, u8 *iv,
1045	struct dm_crypt_request *dmreq)
1046	{
1047	if (bio_data_dir(dmreq->ctx->bio_in) != WRITE)
1048	return crypt_iv_elephant(cc, dmreq);
1049
1050	return 0;
1051	}
1052
1053	static int crypt_iv_elephant_init(struct crypt_config *cc)
1054	{
1055	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1056	int key_offset = cc->key_size - cc->key_extra_size;
1057
1058	return crypto_skcipher_setkey(tfm: elephant->tfm, key: &cc->key[key_offset], keylen: cc->key_extra_size);
1059	}
1060
1061	static int crypt_iv_elephant_wipe(struct crypt_config *cc)
1062	{
1063	struct iv_elephant_private *elephant = &cc->iv_gen_private.elephant;
1064	u8 key[ELEPHANT_MAX_KEY_SIZE];
1065
1066	memset(key, 0, cc->key_extra_size);
1067	return crypto_skcipher_setkey(tfm: elephant->tfm, key, keylen: cc->key_extra_size);
1068	}
1069
1070	static const struct crypt_iv_operations crypt_iv_plain_ops = {
1071	.generator = crypt_iv_plain_gen
1072	};
1073
1074	static const struct crypt_iv_operations crypt_iv_plain64_ops = {
1075	.generator = crypt_iv_plain64_gen
1076	};
1077
1078	static const struct crypt_iv_operations crypt_iv_plain64be_ops = {
1079	.generator = crypt_iv_plain64be_gen
1080	};
1081
1082	static const struct crypt_iv_operations crypt_iv_essiv_ops = {
1083	.generator = crypt_iv_essiv_gen
1084	};
1085
1086	static const struct crypt_iv_operations crypt_iv_benbi_ops = {
1087	.ctr = crypt_iv_benbi_ctr,
1088	.dtr = crypt_iv_benbi_dtr,
1089	.generator = crypt_iv_benbi_gen
1090	};
1091
1092	static const struct crypt_iv_operations crypt_iv_null_ops = {
1093	.generator = crypt_iv_null_gen
1094	};
1095
1096	static const struct crypt_iv_operations crypt_iv_lmk_ops = {
1097	.ctr = crypt_iv_lmk_ctr,
1098	.dtr = crypt_iv_lmk_dtr,
1099	.init = crypt_iv_lmk_init,
1100	.wipe = crypt_iv_lmk_wipe,
1101	.generator = crypt_iv_lmk_gen,
1102	.post = crypt_iv_lmk_post
1103	};
1104
1105	static const struct crypt_iv_operations crypt_iv_tcw_ops = {
1106	.ctr = crypt_iv_tcw_ctr,
1107	.dtr = crypt_iv_tcw_dtr,
1108	.init = crypt_iv_tcw_init,
1109	.wipe = crypt_iv_tcw_wipe,
1110	.generator = crypt_iv_tcw_gen,
1111	.post = crypt_iv_tcw_post
1112	};
1113
1114	static const struct crypt_iv_operations crypt_iv_random_ops = {
1115	.generator = crypt_iv_random_gen
1116	};
1117
1118	static const struct crypt_iv_operations crypt_iv_eboiv_ops = {
1119	.ctr = crypt_iv_eboiv_ctr,
1120	.generator = crypt_iv_eboiv_gen
1121	};
1122
1123	static const struct crypt_iv_operations crypt_iv_elephant_ops = {
1124	.ctr = crypt_iv_elephant_ctr,
1125	.dtr = crypt_iv_elephant_dtr,
1126	.init = crypt_iv_elephant_init,
1127	.wipe = crypt_iv_elephant_wipe,
1128	.generator = crypt_iv_elephant_gen,
1129	.post = crypt_iv_elephant_post
1130	};
1131
1132	/*
1133	* Integrity extensions
1134	*/
1135	static bool crypt_integrity_aead(struct crypt_config *cc)
1136	{
1137	return test_bit(CRYPT_MODE_INTEGRITY_AEAD, &cc->cipher_flags);
1138	}
1139
1140	static bool crypt_integrity_hmac(struct crypt_config *cc)
1141	{
1142	return crypt_integrity_aead(cc) && cc->key_mac_size;
1143	}
1144
1145	/* Get sg containing data */
1146	static struct scatterlist *crypt_get_sg_data(struct crypt_config *cc,
1147	struct scatterlist *sg)
1148	{
1149	if (unlikely(crypt_integrity_aead(cc)))
1150	return &sg[2];
1151
1152	return sg;
1153	}
1154
1155	static int dm_crypt_integrity_io_alloc(struct dm_crypt_io *io, struct bio *bio)
1156	{
1157	struct bio_integrity_payload *bip;
1158	unsigned int tag_len;
1159	int ret;
1160
1161	if (!bio_sectors(bio) || !io->cc->tuple_size)
1162	return 0;
1163
1164	bip = bio_integrity_alloc(bio, GFP_NOIO, nr: 1);
1165	if (IS_ERR(ptr: bip))
1166	return PTR_ERR(ptr: bip);
1167
1168	tag_len = io->cc->tuple_size * (bio_sectors(bio) >> io->cc->sector_shift);
1169
1170	bip->bip_iter.bi_sector = bio->bi_iter.bi_sector;
1171
1172	ret = bio_integrity_add_page(bio, virt_to_page(io->integrity_metadata),
1173	len: tag_len, offset_in_page(io->integrity_metadata));
1174	if (unlikely(ret != tag_len))
1175	return -ENOMEM;
1176
1177	return 0;
1178	}
1179
1180	static int crypt_integrity_ctr(struct crypt_config *cc, struct dm_target *ti)
1181	{
1182	#ifdef CONFIG_BLK_DEV_INTEGRITY
1183	struct blk_integrity *bi = blk_get_integrity(disk: cc->dev->bdev->bd_disk);
1184	struct mapped_device *md = dm_table_get_md(t: ti->table);
1185
1186	/* We require an underlying device with non-PI metadata */
1187	if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) {
1188	ti->error = "Integrity profile not supported.";
1189	return -EINVAL;
1190	}
1191
1192	if (bi->tuple_size < cc->used_tag_size) {
1193	ti->error = "Integrity profile tag size mismatch.";
1194	return -EINVAL;
1195	}
1196	cc->tuple_size = bi->tuple_size;
1197	if (1 << bi->interval_exp != cc->sector_size) {
1198	ti->error = "Integrity profile sector size mismatch.";
1199	return -EINVAL;
1200	}
1201
1202	if (crypt_integrity_aead(cc)) {
1203	cc->integrity_tag_size = cc->used_tag_size - cc->integrity_iv_size;
1204	DMDEBUG("%s: Integrity AEAD, tag size %u, IV size %u.", dm_device_name(md),
1205	cc->integrity_tag_size, cc->integrity_iv_size);
1206
1207	if (crypto_aead_setauthsize(tfm: any_tfm_aead(cc), authsize: cc->integrity_tag_size)) {
1208	ti->error = "Integrity AEAD auth tag size is not supported.";
1209	return -EINVAL;
1210	}
1211	} else if (cc->integrity_iv_size)
1212	DMDEBUG("%s: Additional per-sector space %u bytes for IV.", dm_device_name(md),
1213	cc->integrity_iv_size);
1214
1215	if ((cc->integrity_tag_size + cc->integrity_iv_size) > cc->tuple_size) {
1216	ti->error = "Not enough space for integrity tag in the profile.";
1217	return -EINVAL;
1218	}
1219
1220	return 0;
1221	#else
1222	ti->error = "Integrity profile not supported.";
1223	return -EINVAL;
1224	#endif
1225	}
1226
1227	static void crypt_convert_init(struct crypt_config *cc,
1228	struct convert_context *ctx,
1229	struct bio *bio_out, struct bio *bio_in,
1230	sector_t sector)
1231	{
1232	ctx->bio_in = bio_in;
1233	ctx->bio_out = bio_out;
1234	if (bio_in)
1235	ctx->iter_in = bio_in->bi_iter;
1236	if (bio_out)
1237	ctx->iter_out = bio_out->bi_iter;
1238	ctx->cc_sector = sector + cc->iv_offset;
1239	ctx->tag_offset = 0;
1240	init_completion(x: &ctx->restart);
1241	}
1242
1243	static struct dm_crypt_request *dmreq_of_req(struct crypt_config *cc,
1244	void *req)
1245	{
1246	return (struct dm_crypt_request *)((char *)req + cc->dmreq_start);
1247	}
1248
1249	static void *req_of_dmreq(struct crypt_config *cc, struct dm_crypt_request *dmreq)
1250	{
1251	return (void *)((char *)dmreq - cc->dmreq_start);
1252	}
1253
1254	static u8 *iv_of_dmreq(struct crypt_config *cc,
1255	struct dm_crypt_request *dmreq)
1256	{
1257	if (crypt_integrity_aead(cc))
1258	return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1259	crypto_aead_alignmask(any_tfm_aead(cc)) + 1);
1260	else
1261	return (u8 *)ALIGN((unsigned long)(dmreq + 1),
1262	crypto_skcipher_alignmask(any_tfm(cc)) + 1);
1263	}
1264
1265	static u8 *org_iv_of_dmreq(struct crypt_config *cc,
1266	struct dm_crypt_request *dmreq)
1267	{
1268	return iv_of_dmreq(cc, dmreq) + cc->iv_size;
1269	}
1270
1271	static __le64 *org_sector_of_dmreq(struct crypt_config *cc,
1272	struct dm_crypt_request *dmreq)
1273	{
1274	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size + cc->iv_size;
1275
1276	return (__le64 *) ptr;
1277	}
1278
1279	static unsigned int *org_tag_of_dmreq(struct crypt_config *cc,
1280	struct dm_crypt_request *dmreq)
1281	{
1282	u8 *ptr = iv_of_dmreq(cc, dmreq) + cc->iv_size +
1283	cc->iv_size + sizeof(uint64_t);
1284
1285	return (unsigned int *)ptr;
1286	}
1287
1288	static void *tag_from_dmreq(struct crypt_config *cc,
1289	struct dm_crypt_request *dmreq)
1290	{
1291	struct convert_context *ctx = dmreq->ctx;
1292	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
1293
1294	return &io->integrity_metadata[*org_tag_of_dmreq(cc, dmreq) *
1295	cc->tuple_size];
1296	}
1297
1298	static void *iv_tag_from_dmreq(struct crypt_config *cc,
1299	struct dm_crypt_request *dmreq)
1300	{
1301	return tag_from_dmreq(cc, dmreq) + cc->integrity_tag_size;
1302	}
1303
1304	static int crypt_convert_block_aead(struct crypt_config *cc,
1305	struct convert_context *ctx,
1306	struct aead_request *req,
1307	unsigned int tag_offset)
1308	{
1309	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1310	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1311	struct dm_crypt_request *dmreq;
1312	u8 *iv, *org_iv, *tag_iv, *tag;
1313	__le64 *sector;
1314	int r = 0;
1315
1316	BUG_ON(cc->integrity_iv_size && cc->integrity_iv_size != cc->iv_size);
1317
1318	/* Reject unexpected unaligned bio. */
1319	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1320	return -EIO;
1321
1322	dmreq = dmreq_of_req(cc, req);
1323	dmreq->iv_sector = ctx->cc_sector;
1324	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1325	dmreq->iv_sector >>= cc->sector_shift;
1326	dmreq->ctx = ctx;
1327
1328	*org_tag_of_dmreq(cc, dmreq) = tag_offset;
1329
1330	sector = org_sector_of_dmreq(cc, dmreq);
1331	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1332
1333	iv = iv_of_dmreq(cc, dmreq);
1334	org_iv = org_iv_of_dmreq(cc, dmreq);
1335	tag = tag_from_dmreq(cc, dmreq);
1336	tag_iv = iv_tag_from_dmreq(cc, dmreq);
1337
1338	/* AEAD request:
1339	* |----- AAD -------|------ DATA -------|-- AUTH TAG --|
1340	* | (authenticated) | (auth+encryption) | |
1341	* | sector_LE | IV | sector in/out | tag in/out |
1342	*/
1343	sg_init_table(dmreq->sg_in, 4);
1344	sg_set_buf(sg: &dmreq->sg_in[0], buf: sector, buflen: sizeof(uint64_t));
1345	sg_set_buf(sg: &dmreq->sg_in[1], buf: org_iv, buflen: cc->iv_size);
1346	sg_set_page(sg: &dmreq->sg_in[2], page: bv_in.bv_page, len: cc->sector_size, offset: bv_in.bv_offset);
1347	sg_set_buf(sg: &dmreq->sg_in[3], buf: tag, buflen: cc->integrity_tag_size);
1348
1349	sg_init_table(dmreq->sg_out, 4);
1350	sg_set_buf(sg: &dmreq->sg_out[0], buf: sector, buflen: sizeof(uint64_t));
1351	sg_set_buf(sg: &dmreq->sg_out[1], buf: org_iv, buflen: cc->iv_size);
1352	sg_set_page(sg: &dmreq->sg_out[2], page: bv_out.bv_page, len: cc->sector_size, offset: bv_out.bv_offset);
1353	sg_set_buf(sg: &dmreq->sg_out[3], buf: tag, buflen: cc->integrity_tag_size);
1354
1355	if (cc->iv_gen_ops) {
1356	/* For READs use IV stored in integrity metadata */
1357	if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1358	memcpy(org_iv, tag_iv, cc->iv_size);
1359	} else {
1360	r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1361	if (r < 0)
1362	return r;
1363	/* Store generated IV in integrity metadata */
1364	if (cc->integrity_iv_size)
1365	memcpy(tag_iv, org_iv, cc->iv_size);
1366	}
1367	/* Working copy of IV, to be modified in crypto API */
1368	memcpy(iv, org_iv, cc->iv_size);
1369	}
1370
1371	aead_request_set_ad(req, assoclen: sizeof(uint64_t) + cc->iv_size);
1372	if (bio_data_dir(ctx->bio_in) == WRITE) {
1373	aead_request_set_crypt(req, src: dmreq->sg_in, dst: dmreq->sg_out,
1374	cryptlen: cc->sector_size, iv);
1375	r = crypto_aead_encrypt(req);
1376	if (cc->integrity_tag_size + cc->integrity_iv_size != cc->tuple_size)
1377	memset(tag + cc->integrity_tag_size + cc->integrity_iv_size, 0,
1378	cc->tuple_size - (cc->integrity_tag_size + cc->integrity_iv_size));
1379	} else {
1380	aead_request_set_crypt(req, src: dmreq->sg_in, dst: dmreq->sg_out,
1381	cryptlen: cc->sector_size + cc->integrity_tag_size, iv);
1382	r = crypto_aead_decrypt(req);
1383	}
1384
1385	if (r == -EBADMSG) {
1386	sector_t s = le64_to_cpu(*sector);
1387
1388	ctx->aead_failed = true;
1389	if (ctx->aead_recheck) {
1390	DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
1391	ctx->bio_in->bi_bdev, s);
1392	dm_audit_log_bio(DM_MSG_PREFIX, op: "integrity-aead",
1393	bio: ctx->bio_in, sector: s, result: 0);
1394	}
1395	}
1396
1397	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1398	r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1399
1400	bio_advance_iter(bio: ctx->bio_in, iter: &ctx->iter_in, bytes: cc->sector_size);
1401	bio_advance_iter(bio: ctx->bio_out, iter: &ctx->iter_out, bytes: cc->sector_size);
1402
1403	return r;
1404	}
1405
1406	static int crypt_convert_block_skcipher(struct crypt_config *cc,
1407	struct convert_context *ctx,
1408	struct skcipher_request *req,
1409	unsigned int tag_offset)
1410	{
1411	struct bio_vec bv_in = bio_iter_iovec(ctx->bio_in, ctx->iter_in);
1412	struct bio_vec bv_out = bio_iter_iovec(ctx->bio_out, ctx->iter_out);
1413	struct scatterlist *sg_in, *sg_out;
1414	struct dm_crypt_request *dmreq;
1415	u8 *iv, *org_iv, *tag_iv;
1416	__le64 *sector;
1417	int r = 0;
1418
1419	/* Reject unexpected unaligned bio. */
1420	if (unlikely(bv_in.bv_len & (cc->sector_size - 1)))
1421	return -EIO;
1422
1423	dmreq = dmreq_of_req(cc, req);
1424	dmreq->iv_sector = ctx->cc_sector;
1425	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
1426	dmreq->iv_sector >>= cc->sector_shift;
1427	dmreq->ctx = ctx;
1428
1429	*org_tag_of_dmreq(cc, dmreq) = tag_offset;
1430
1431	iv = iv_of_dmreq(cc, dmreq);
1432	org_iv = org_iv_of_dmreq(cc, dmreq);
1433	tag_iv = iv_tag_from_dmreq(cc, dmreq);
1434
1435	sector = org_sector_of_dmreq(cc, dmreq);
1436	*sector = cpu_to_le64(ctx->cc_sector - cc->iv_offset);
1437
1438	/* For skcipher we use only the first sg item */
1439	sg_in = &dmreq->sg_in[0];
1440	sg_out = &dmreq->sg_out[0];
1441
1442	sg_init_table(sg_in, 1);
1443	sg_set_page(sg: sg_in, page: bv_in.bv_page, len: cc->sector_size, offset: bv_in.bv_offset);
1444
1445	sg_init_table(sg_out, 1);
1446	sg_set_page(sg: sg_out, page: bv_out.bv_page, len: cc->sector_size, offset: bv_out.bv_offset);
1447
1448	if (cc->iv_gen_ops) {
1449	/* For READs use IV stored in integrity metadata */
1450	if (cc->integrity_iv_size && bio_data_dir(ctx->bio_in) != WRITE) {
1451	memcpy(org_iv, tag_iv, cc->integrity_iv_size);
1452	} else {
1453	r = cc->iv_gen_ops->generator(cc, org_iv, dmreq);
1454	if (r < 0)
1455	return r;
1456	/* Data can be already preprocessed in generator */
1457	if (test_bit(CRYPT_ENCRYPT_PREPROCESS, &cc->cipher_flags))
1458	sg_in = sg_out;
1459	/* Store generated IV in integrity metadata */
1460	if (cc->integrity_iv_size)
1461	memcpy(tag_iv, org_iv, cc->integrity_iv_size);
1462	}
1463	/* Working copy of IV, to be modified in crypto API */
1464	memcpy(iv, org_iv, cc->iv_size);
1465	}
1466
1467	skcipher_request_set_crypt(req, src: sg_in, dst: sg_out, cryptlen: cc->sector_size, iv);
1468
1469	if (bio_data_dir(ctx->bio_in) == WRITE)
1470	r = crypto_skcipher_encrypt(req);
1471	else
1472	r = crypto_skcipher_decrypt(req);
1473
1474	if (!r && cc->iv_gen_ops && cc->iv_gen_ops->post)
1475	r = cc->iv_gen_ops->post(cc, org_iv, dmreq);
1476
1477	bio_advance_iter(bio: ctx->bio_in, iter: &ctx->iter_in, bytes: cc->sector_size);
1478	bio_advance_iter(bio: ctx->bio_out, iter: &ctx->iter_out, bytes: cc->sector_size);
1479
1480	return r;
1481	}
1482
1483	static void kcryptd_async_done(void *async_req, int error);
1484
1485	static int crypt_alloc_req_skcipher(struct crypt_config *cc,
1486	struct convert_context *ctx)
1487	{
1488	unsigned int key_index = ctx->cc_sector & (cc->tfms_count - 1);
1489
1490	if (!ctx->r.req) {
1491	ctx->r.req = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1492	if (!ctx->r.req)
1493	return -ENOMEM;
1494	}
1495
1496	skcipher_request_set_tfm(req: ctx->r.req, tfm: cc->cipher_tfm.tfms[key_index]);
1497
1498	/*
1499	* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1500	* requests if driver request queue is full.
1501	*/
1502	skcipher_request_set_callback(req: ctx->r.req,
1503	CRYPTO_TFM_REQ_MAY_BACKLOG,
1504	compl: kcryptd_async_done, data: dmreq_of_req(cc, req: ctx->r.req));
1505
1506	return 0;
1507	}
1508
1509	static int crypt_alloc_req_aead(struct crypt_config *cc,
1510	struct convert_context *ctx)
1511	{
1512	if (!ctx->r.req_aead) {
1513	ctx->r.req_aead = mempool_alloc(&cc->req_pool, in_interrupt() ? GFP_ATOMIC : GFP_NOIO);
1514	if (!ctx->r.req_aead)
1515	return -ENOMEM;
1516	}
1517
1518	aead_request_set_tfm(req: ctx->r.req_aead, tfm: cc->cipher_tfm.tfms_aead[0]);
1519
1520	/*
1521	* Use REQ_MAY_BACKLOG so a cipher driver internally backlogs
1522	* requests if driver request queue is full.
1523	*/
1524	aead_request_set_callback(req: ctx->r.req_aead,
1525	CRYPTO_TFM_REQ_MAY_BACKLOG,
1526	compl: kcryptd_async_done, data: dmreq_of_req(cc, req: ctx->r.req_aead));
1527
1528	return 0;
1529	}
1530
1531	static int crypt_alloc_req(struct crypt_config *cc,
1532	struct convert_context *ctx)
1533	{
1534	if (crypt_integrity_aead(cc))
1535	return crypt_alloc_req_aead(cc, ctx);
1536	else
1537	return crypt_alloc_req_skcipher(cc, ctx);
1538	}
1539
1540	static void crypt_free_req_skcipher(struct crypt_config *cc,
1541	struct skcipher_request *req, struct bio *base_bio)
1542	{
1543	struct dm_crypt_io *io = dm_per_bio_data(bio: base_bio, data_size: cc->per_bio_data_size);
1544
1545	if ((struct skcipher_request *)(io + 1) != req)
1546	mempool_free(element: req, pool: &cc->req_pool);
1547	}
1548
1549	static void crypt_free_req_aead(struct crypt_config *cc,
1550	struct aead_request *req, struct bio *base_bio)
1551	{
1552	struct dm_crypt_io *io = dm_per_bio_data(bio: base_bio, data_size: cc->per_bio_data_size);
1553
1554	if ((struct aead_request *)(io + 1) != req)
1555	mempool_free(element: req, pool: &cc->req_pool);
1556	}
1557
1558	static void crypt_free_req(struct crypt_config *cc, void *req, struct bio *base_bio)
1559	{
1560	if (crypt_integrity_aead(cc))
1561	crypt_free_req_aead(cc, req, base_bio);
1562	else
1563	crypt_free_req_skcipher(cc, req, base_bio);
1564	}
1565
1566	/*
1567	* Encrypt / decrypt data from one bio to another one (can be the same one)
1568	*/
1569	static blk_status_t crypt_convert(struct crypt_config *cc,
1570	struct convert_context *ctx, bool atomic, bool reset_pending)
1571	{
1572	unsigned int sector_step = cc->sector_size >> SECTOR_SHIFT;
1573	int r;
1574
1575	/*
1576	* if reset_pending is set we are dealing with the bio for the first time,
1577	* else we're continuing to work on the previous bio, so don't mess with
1578	* the cc_pending counter
1579	*/
1580	if (reset_pending)
1581	atomic_set(v: &ctx->cc_pending, i: 1);
1582
1583	while (ctx->iter_in.bi_size && ctx->iter_out.bi_size) {
1584
1585	r = crypt_alloc_req(cc, ctx);
1586	if (r) {
1587	complete(&ctx->restart);
1588	return BLK_STS_DEV_RESOURCE;
1589	}
1590
1591	atomic_inc(v: &ctx->cc_pending);
1592
1593	if (crypt_integrity_aead(cc))
1594	r = crypt_convert_block_aead(cc, ctx, req: ctx->r.req_aead, tag_offset: ctx->tag_offset);
1595	else
1596	r = crypt_convert_block_skcipher(cc, ctx, req: ctx->r.req, tag_offset: ctx->tag_offset);
1597
1598	switch (r) {
1599	/*
1600	* The request was queued by a crypto driver
1601	* but the driver request queue is full, let's wait.
1602	*/
1603	case -EBUSY:
1604	if (in_interrupt()) {
1605	if (try_wait_for_completion(x: &ctx->restart)) {
1606	/*
1607	* we don't have to block to wait for completion,
1608	* so proceed
1609	*/
1610	} else {
1611	/*
1612	* we can't wait for completion without blocking
1613	* exit and continue processing in a workqueue
1614	*/
1615	ctx->r.req = NULL;
1616	ctx->tag_offset++;
1617	ctx->cc_sector += sector_step;
1618	return BLK_STS_DEV_RESOURCE;
1619	}
1620	} else {
1621	wait_for_completion(&ctx->restart);
1622	}
1623	reinit_completion(x: &ctx->restart);
1624	fallthrough;
1625	/*
1626	* The request is queued and processed asynchronously,
1627	* completion function kcryptd_async_done() will be called.
1628	*/
1629	case -EINPROGRESS:
1630	ctx->r.req = NULL;
1631	ctx->tag_offset++;
1632	ctx->cc_sector += sector_step;
1633	continue;
1634	/*
1635	* The request was already processed (synchronously).
1636	*/
1637	case 0:
1638	atomic_dec(v: &ctx->cc_pending);
1639	ctx->cc_sector += sector_step;
1640	ctx->tag_offset++;
1641	if (!atomic)
1642	cond_resched();
1643	continue;
1644	/*
1645	* There was a data integrity error.
1646	*/
1647	case -EBADMSG:
1648	atomic_dec(v: &ctx->cc_pending);
1649	return BLK_STS_PROTECTION;
1650	/*
1651	* There was an error while processing the request.
1652	*/
1653	default:
1654	atomic_dec(v: &ctx->cc_pending);
1655	return BLK_STS_IOERR;
1656	}
1657	}
1658
1659	return 0;
1660	}
1661
1662	static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone);
1663
1664	/*
1665	* Generate a new unfragmented bio with the given size
1666	* This should never violate the device limitations (but if it did then block
1667	* core should split the bio as needed).
1668	*
1669	* This function may be called concurrently. If we allocate from the mempool
1670	* concurrently, there is a possibility of deadlock. For example, if we have
1671	* mempool of 256 pages, two processes, each wanting 256, pages allocate from
1672	* the mempool concurrently, it may deadlock in a situation where both processes
1673	* have allocated 128 pages and the mempool is exhausted.
1674	*
1675	* In order to avoid this scenario we allocate the pages under a mutex.
1676	*
1677	* In order to not degrade performance with excessive locking, we try
1678	* non-blocking allocations without a mutex first but on failure we fallback
1679	* to blocking allocations with a mutex.
1680	*
1681	* In order to reduce allocation overhead, we try to allocate compound pages in
1682	* the first pass. If they are not available, we fall back to the mempool.
1683	*/
1684	static struct bio *crypt_alloc_buffer(struct dm_crypt_io *io, unsigned int size)
1685	{
1686	struct crypt_config *cc = io->cc;
1687	struct bio *clone;
1688	unsigned int nr_iovecs = (size + PAGE_SIZE - 1) >> PAGE_SHIFT;
1689	gfp_t gfp_mask = GFP_NOWAIT | __GFP_HIGHMEM;
1690	unsigned int remaining_size;
1691	unsigned int order = MAX_PAGE_ORDER;
1692
1693	retry:
1694	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1695	mutex_lock(&cc->bio_alloc_lock);
1696
1697	clone = bio_alloc_bioset(bdev: cc->dev->bdev, nr_vecs: nr_iovecs, opf: io->base_bio->bi_opf,
1698	GFP_NOIO, bs: &cc->bs);
1699	clone->bi_private = io;
1700	clone->bi_end_io = crypt_endio;
1701	clone->bi_ioprio = io->base_bio->bi_ioprio;
1702	clone->bi_iter.bi_sector = cc->start + io->sector;
1703
1704	remaining_size = size;
1705
1706	while (remaining_size) {
1707	struct page *pages;
1708	unsigned size_to_add;
1709	unsigned remaining_order = __fls(word: (remaining_size + PAGE_SIZE - 1) >> PAGE_SHIFT);
1710	order = min(order, remaining_order);
1711
1712	while (order > 0) {
1713	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) +
1714	(1 << order) > dm_crypt_pages_per_client))
1715	goto decrease_order;
1716	pages = alloc_pages(gfp_mask
1717	| __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN | __GFP_COMP,
1718	order);
1719	if (likely(pages != NULL)) {
1720	percpu_counter_add(fbc: &cc->n_allocated_pages, amount: 1 << order);
1721	goto have_pages;
1722	}
1723	decrease_order:
1724	order--;
1725	}
1726
1727	pages = mempool_alloc(&cc->page_pool, gfp_mask);
1728	if (!pages) {
1729	crypt_free_buffer_pages(cc, clone);
1730	bio_put(clone);
1731	gfp_mask |= __GFP_DIRECT_RECLAIM;
1732	order = 0;
1733	goto retry;
1734	}
1735
1736	have_pages:
1737	size_to_add = min((unsigned)PAGE_SIZE << order, remaining_size);
1738	__bio_add_page(bio: clone, page: pages, len: size_to_add, off: 0);
1739	remaining_size -= size_to_add;
1740	}
1741
1742	/* Allocate space for integrity tags */
1743	if (dm_crypt_integrity_io_alloc(io, bio: clone)) {
1744	crypt_free_buffer_pages(cc, clone);
1745	bio_put(clone);
1746	clone = NULL;
1747	}
1748
1749	if (unlikely(gfp_mask & __GFP_DIRECT_RECLAIM))
1750	mutex_unlock(lock: &cc->bio_alloc_lock);
1751
1752	return clone;
1753	}
1754
1755	static void crypt_free_buffer_pages(struct crypt_config *cc, struct bio *clone)
1756	{
1757	struct folio_iter fi;
1758
1759	if (clone->bi_vcnt > 0) { /* bio_for_each_folio_all crashes with an empty bio */
1760	bio_for_each_folio_all(fi, clone) {
1761	if (folio_test_large(folio: fi.folio)) {
1762	percpu_counter_sub(fbc: &cc->n_allocated_pages,
1763	amount: 1 << folio_order(folio: fi.folio));
1764	folio_put(folio: fi.folio);
1765	} else {
1766	mempool_free(element: &fi.folio->page, pool: &cc->page_pool);
1767	}
1768	}
1769	}
1770	}
1771
1772	static void crypt_io_init(struct dm_crypt_io *io, struct crypt_config *cc,
1773	struct bio *bio, sector_t sector)
1774	{
1775	io->cc = cc;
1776	io->base_bio = bio;
1777	io->sector = sector;
1778	io->error = 0;
1779	io->ctx.aead_recheck = false;
1780	io->ctx.aead_failed = false;
1781	io->ctx.r.req = NULL;
1782	io->integrity_metadata = NULL;
1783	io->integrity_metadata_from_pool = false;
1784	atomic_set(v: &io->io_pending, i: 0);
1785	}
1786
1787	static void crypt_inc_pending(struct dm_crypt_io *io)
1788	{
1789	atomic_inc(v: &io->io_pending);
1790	}
1791
1792	static void kcryptd_queue_read(struct dm_crypt_io *io);
1793
1794	/*
1795	* One of the bios was finished. Check for completion of
1796	* the whole request and correctly clean up the buffer.
1797	*/
1798	static void crypt_dec_pending(struct dm_crypt_io *io)
1799	{
1800	struct crypt_config *cc = io->cc;
1801	struct bio *base_bio = io->base_bio;
1802	blk_status_t error = io->error;
1803
1804	if (!atomic_dec_and_test(v: &io->io_pending))
1805	return;
1806
1807	if (likely(!io->ctx.aead_recheck) && unlikely(io->ctx.aead_failed) &&
1808	cc->used_tag_size && bio_data_dir(base_bio) == READ) {
1809	io->ctx.aead_recheck = true;
1810	io->ctx.aead_failed = false;
1811	io->error = 0;
1812	kcryptd_queue_read(io);
1813	return;
1814	}
1815
1816	if (io->ctx.r.req)
1817	crypt_free_req(cc, req: io->ctx.r.req, base_bio);
1818
1819	if (unlikely(io->integrity_metadata_from_pool))
1820	mempool_free(element: io->integrity_metadata, pool: &io->cc->tag_pool);
1821	else
1822	kfree(objp: io->integrity_metadata);
1823
1824	base_bio->bi_status = error;
1825
1826	bio_endio(base_bio);
1827	}
1828
1829	/*
1830	* kcryptd/kcryptd_io:
1831	*
1832	* Needed because it would be very unwise to do decryption in an
1833	* interrupt context.
1834	*
1835	* kcryptd performs the actual encryption or decryption.
1836	*
1837	* kcryptd_io performs the IO submission.
1838	*
1839	* They must be separated as otherwise the final stages could be
1840	* starved by new requests which can block in the first stages due
1841	* to memory allocation.
1842	*
1843	* The work is done per CPU global for all dm-crypt instances.
1844	* They should not depend on each other and do not block.
1845	*/
1846	static void crypt_endio(struct bio *clone)
1847	{
1848	struct dm_crypt_io *io = clone->bi_private;
1849	struct crypt_config *cc = io->cc;
1850	unsigned int rw = bio_data_dir(clone);
1851	blk_status_t error = clone->bi_status;
1852
1853	if (io->ctx.aead_recheck && !error) {
1854	kcryptd_queue_crypt(io);
1855	return;
1856	}
1857
1858	/*
1859	* free the processed pages
1860	*/
1861	if (rw == WRITE || io->ctx.aead_recheck)
1862	crypt_free_buffer_pages(cc, clone);
1863
1864	bio_put(clone);
1865
1866	if (rw == READ && !error) {
1867	kcryptd_queue_crypt(io);
1868	return;
1869	}
1870
1871	if (unlikely(error))
1872	io->error = error;
1873
1874	crypt_dec_pending(io);
1875	}
1876
1877	#define CRYPT_MAP_READ_GFP GFP_NOWAIT
1878
1879	static int kcryptd_io_read(struct dm_crypt_io *io, gfp_t gfp)
1880	{
1881	struct crypt_config *cc = io->cc;
1882	struct bio *clone;
1883
1884	if (io->ctx.aead_recheck) {
1885	if (!(gfp & __GFP_DIRECT_RECLAIM))
1886	return 1;
1887	crypt_inc_pending(io);
1888	clone = crypt_alloc_buffer(io, size: io->base_bio->bi_iter.bi_size);
1889	if (unlikely(!clone)) {
1890	crypt_dec_pending(io);
1891	return 1;
1892	}
1893	crypt_convert_init(cc, ctx: &io->ctx, bio_out: clone, bio_in: clone, sector: io->sector);
1894	io->saved_bi_iter = clone->bi_iter;
1895	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
1896	return 0;
1897	}
1898
1899	/*
1900	* We need the original biovec array in order to decrypt the whole bio
1901	* data *afterwards* -- thanks to immutable biovecs we don't need to
1902	* worry about the block layer modifying the biovec array; so leverage
1903	* bio_alloc_clone().
1904	*/
1905	clone = bio_alloc_clone(bdev: cc->dev->bdev, bio_src: io->base_bio, gfp, bs: &cc->bs);
1906	if (!clone)
1907	return 1;
1908
1909	clone->bi_iter.bi_sector = cc->start + io->sector;
1910	clone->bi_private = io;
1911	clone->bi_end_io = crypt_endio;
1912
1913	crypt_inc_pending(io);
1914
1915	if (dm_crypt_integrity_io_alloc(io, bio: clone)) {
1916	crypt_dec_pending(io);
1917	bio_put(clone);
1918	return 1;
1919	}
1920
1921	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
1922	return 0;
1923	}
1924
1925	static void kcryptd_io_read_work(struct work_struct *work)
1926	{
1927	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
1928
1929	crypt_inc_pending(io);
1930	if (kcryptd_io_read(io, GFP_NOIO))
1931	io->error = BLK_STS_RESOURCE;
1932	crypt_dec_pending(io);
1933	}
1934
1935	static void kcryptd_queue_read(struct dm_crypt_io *io)
1936	{
1937	struct crypt_config *cc = io->cc;
1938
1939	INIT_WORK(&io->work, kcryptd_io_read_work);
1940	queue_work(wq: cc->io_queue, work: &io->work);
1941	}
1942
1943	static void kcryptd_io_write(struct dm_crypt_io *io)
1944	{
1945	struct bio *clone = io->ctx.bio_out;
1946
1947	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
1948	}
1949
1950	#define crypt_io_from_node(node) rb_entry((node), struct dm_crypt_io, rb_node)
1951
1952	static int dmcrypt_write(void *data)
1953	{
1954	struct crypt_config *cc = data;
1955	struct dm_crypt_io *io;
1956
1957	while (1) {
1958	struct rb_root write_tree;
1959	struct blk_plug plug;
1960
1961	spin_lock_irq(lock: &cc->write_thread_lock);
1962	continue_locked:
1963
1964	if (!RB_EMPTY_ROOT(&cc->write_tree))
1965	goto pop_from_list;
1966
1967	set_current_state(TASK_INTERRUPTIBLE);
1968
1969	spin_unlock_irq(lock: &cc->write_thread_lock);
1970
1971	if (unlikely(kthread_should_stop())) {
1972	set_current_state(TASK_RUNNING);
1973	break;
1974	}
1975
1976	schedule();
1977
1978	spin_lock_irq(lock: &cc->write_thread_lock);
1979	goto continue_locked;
1980
1981	pop_from_list:
1982	write_tree = cc->write_tree;
1983	cc->write_tree = RB_ROOT;
1984	spin_unlock_irq(lock: &cc->write_thread_lock);
1985
1986	BUG_ON(rb_parent(write_tree.rb_node));
1987
1988	/*
1989	* Note: we cannot walk the tree here with rb_next because
1990	* the structures may be freed when kcryptd_io_write is called.
1991	*/
1992	blk_start_plug(&plug);
1993	do {
1994	io = crypt_io_from_node(rb_first(&write_tree));
1995	rb_erase(&io->rb_node, &write_tree);
1996	kcryptd_io_write(io);
1997	cond_resched();
1998	} while (!RB_EMPTY_ROOT(&write_tree));
1999	blk_finish_plug(&plug);
2000	}
2001	return 0;
2002	}
2003
2004	static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async)
2005	{
2006	struct bio *clone = io->ctx.bio_out;
2007	struct crypt_config *cc = io->cc;
2008	unsigned long flags;
2009	sector_t sector;
2010	struct rb_node **rbp, *parent;
2011
2012	if (unlikely(io->error)) {
2013	crypt_free_buffer_pages(cc, clone);
2014	bio_put(clone);
2015	crypt_dec_pending(io);
2016	return;
2017	}
2018
2019	/* crypt_convert should have filled the clone bio */
2020	BUG_ON(io->ctx.iter_out.bi_size);
2021
2022	if ((likely(!async) && test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags)) ||
2023	test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags)) {
2024	dm_submit_bio_remap(clone: io->base_bio, tgt_clone: clone);
2025	return;
2026	}
2027
2028	spin_lock_irqsave(&cc->write_thread_lock, flags);
2029	if (RB_EMPTY_ROOT(&cc->write_tree))
2030	wake_up_process(tsk: cc->write_thread);
2031	rbp = &cc->write_tree.rb_node;
2032	parent = NULL;
2033	sector = io->sector;
2034	while (*rbp) {
2035	parent = *rbp;
2036	if (sector < crypt_io_from_node(parent)->sector)
2037	rbp = &(*rbp)->rb_left;
2038	else
2039	rbp = &(*rbp)->rb_right;
2040	}
2041	rb_link_node(node: &io->rb_node, parent, rb_link: rbp);
2042	rb_insert_color(&io->rb_node, &cc->write_tree);
2043	spin_unlock_irqrestore(lock: &cc->write_thread_lock, flags);
2044	}
2045
2046	static bool kcryptd_crypt_write_inline(struct crypt_config *cc,
2047	struct convert_context *ctx)
2048
2049	{
2050	if (!test_bit(DM_CRYPT_WRITE_INLINE, &cc->flags))
2051	return false;
2052
2053	/*
2054	* Note: zone append writes (REQ_OP_ZONE_APPEND) do not have ordering
2055	* constraints so they do not need to be issued inline by
2056	* kcryptd_crypt_write_convert().
2057	*/
2058	switch (bio_op(bio: ctx->bio_in)) {
2059	case REQ_OP_WRITE:
2060	case REQ_OP_WRITE_ZEROES:
2061	return true;
2062	default:
2063	return false;
2064	}
2065	}
2066
2067	static void kcryptd_crypt_write_continue(struct work_struct *work)
2068	{
2069	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2070	struct crypt_config *cc = io->cc;
2071	struct convert_context *ctx = &io->ctx;
2072	int crypt_finished;
2073	blk_status_t r;
2074
2075	wait_for_completion(&ctx->restart);
2076	reinit_completion(x: &ctx->restart);
2077
2078	r = crypt_convert(cc, ctx: &io->ctx, atomic: false, reset_pending: false);
2079	if (r)
2080	io->error = r;
2081	crypt_finished = atomic_dec_and_test(v: &ctx->cc_pending);
2082	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2083	/* Wait for completion signaled by kcryptd_async_done() */
2084	wait_for_completion(&ctx->restart);
2085	crypt_finished = 1;
2086	}
2087
2088	/* Encryption was already finished, submit io now */
2089	if (crypt_finished)
2090	kcryptd_crypt_write_io_submit(io, async: 0);
2091
2092	crypt_dec_pending(io);
2093	}
2094
2095	static void kcryptd_crypt_write_convert(struct dm_crypt_io *io)
2096	{
2097	struct crypt_config *cc = io->cc;
2098	struct convert_context *ctx = &io->ctx;
2099	struct bio *clone;
2100	int crypt_finished;
2101	blk_status_t r;
2102
2103	/*
2104	* Prevent io from disappearing until this function completes.
2105	*/
2106	crypt_inc_pending(io);
2107	crypt_convert_init(cc, ctx, NULL, bio_in: io->base_bio, sector: io->sector);
2108
2109	clone = crypt_alloc_buffer(io, size: io->base_bio->bi_iter.bi_size);
2110	if (unlikely(!clone)) {
2111	io->error = BLK_STS_IOERR;
2112	goto dec;
2113	}
2114
2115	io->ctx.bio_out = clone;
2116	io->ctx.iter_out = clone->bi_iter;
2117
2118	if (crypt_integrity_aead(cc)) {
2119	bio_copy_data(dst: clone, src: io->base_bio);
2120	io->ctx.bio_in = clone;
2121	io->ctx.iter_in = clone->bi_iter;
2122	}
2123
2124	crypt_inc_pending(io);
2125	r = crypt_convert(cc, ctx,
2126	test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags), reset_pending: true);
2127	/*
2128	* Crypto API backlogged the request, because its queue was full
2129	* and we're in softirq context, so continue from a workqueue
2130	* (TODO: is it actually possible to be in softirq in the write path?)
2131	*/
2132	if (r == BLK_STS_DEV_RESOURCE) {
2133	INIT_WORK(&io->work, kcryptd_crypt_write_continue);
2134	queue_work(wq: cc->crypt_queue, work: &io->work);
2135	return;
2136	}
2137	if (r)
2138	io->error = r;
2139	crypt_finished = atomic_dec_and_test(v: &ctx->cc_pending);
2140	if (!crypt_finished && kcryptd_crypt_write_inline(cc, ctx)) {
2141	/* Wait for completion signaled by kcryptd_async_done() */
2142	wait_for_completion(&ctx->restart);
2143	crypt_finished = 1;
2144	}
2145
2146	/* Encryption was already finished, submit io now */
2147	if (crypt_finished)
2148	kcryptd_crypt_write_io_submit(io, async: 0);
2149
2150	dec:
2151	crypt_dec_pending(io);
2152	}
2153
2154	static void kcryptd_crypt_read_done(struct dm_crypt_io *io)
2155	{
2156	if (io->ctx.aead_recheck) {
2157	if (!io->error) {
2158	io->ctx.bio_in->bi_iter = io->saved_bi_iter;
2159	bio_copy_data(dst: io->base_bio, src: io->ctx.bio_in);
2160	}
2161	crypt_free_buffer_pages(cc: io->cc, clone: io->ctx.bio_in);
2162	bio_put(io->ctx.bio_in);
2163	}
2164	crypt_dec_pending(io);
2165	}
2166
2167	static void kcryptd_crypt_read_continue(struct work_struct *work)
2168	{
2169	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2170	struct crypt_config *cc = io->cc;
2171	blk_status_t r;
2172
2173	wait_for_completion(&io->ctx.restart);
2174	reinit_completion(x: &io->ctx.restart);
2175
2176	r = crypt_convert(cc, ctx: &io->ctx, atomic: false, reset_pending: false);
2177	if (r)
2178	io->error = r;
2179
2180	if (atomic_dec_and_test(v: &io->ctx.cc_pending))
2181	kcryptd_crypt_read_done(io);
2182
2183	crypt_dec_pending(io);
2184	}
2185
2186	static void kcryptd_crypt_read_convert(struct dm_crypt_io *io)
2187	{
2188	struct crypt_config *cc = io->cc;
2189	blk_status_t r;
2190
2191	crypt_inc_pending(io);
2192
2193	if (io->ctx.aead_recheck) {
2194	r = crypt_convert(cc, ctx: &io->ctx,
2195	test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), reset_pending: true);
2196	} else {
2197	crypt_convert_init(cc, ctx: &io->ctx, bio_out: io->base_bio, bio_in: io->base_bio,
2198	sector: io->sector);
2199
2200	r = crypt_convert(cc, ctx: &io->ctx,
2201	test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags), reset_pending: true);
2202	}
2203	/*
2204	* Crypto API backlogged the request, because its queue was full
2205	* and we're in softirq context, so continue from a workqueue
2206	*/
2207	if (r == BLK_STS_DEV_RESOURCE) {
2208	INIT_WORK(&io->work, kcryptd_crypt_read_continue);
2209	queue_work(wq: cc->crypt_queue, work: &io->work);
2210	return;
2211	}
2212	if (r)
2213	io->error = r;
2214
2215	if (atomic_dec_and_test(v: &io->ctx.cc_pending))
2216	kcryptd_crypt_read_done(io);
2217
2218	crypt_dec_pending(io);
2219	}
2220
2221	static void kcryptd_async_done(void *data, int error)
2222	{
2223	struct dm_crypt_request *dmreq = data;
2224	struct convert_context *ctx = dmreq->ctx;
2225	struct dm_crypt_io *io = container_of(ctx, struct dm_crypt_io, ctx);
2226	struct crypt_config *cc = io->cc;
2227
2228	/*
2229	* A request from crypto driver backlog is going to be processed now,
2230	* finish the completion and continue in crypt_convert().
2231	* (Callback will be called for the second time for this request.)
2232	*/
2233	if (error == -EINPROGRESS) {
2234	complete(&ctx->restart);
2235	return;
2236	}
2237
2238	if (!error && cc->iv_gen_ops && cc->iv_gen_ops->post)
2239	error = cc->iv_gen_ops->post(cc, org_iv_of_dmreq(cc, dmreq), dmreq);
2240
2241	if (error == -EBADMSG) {
2242	sector_t s = le64_to_cpu(*org_sector_of_dmreq(cc, dmreq));
2243
2244	ctx->aead_failed = true;
2245	if (ctx->aead_recheck) {
2246	DMERR_LIMIT("%pg: INTEGRITY AEAD ERROR, sector %llu",
2247	ctx->bio_in->bi_bdev, s);
2248	dm_audit_log_bio(DM_MSG_PREFIX, op: "integrity-aead",
2249	bio: ctx->bio_in, sector: s, result: 0);
2250	}
2251	io->error = BLK_STS_PROTECTION;
2252	} else if (error < 0)
2253	io->error = BLK_STS_IOERR;
2254
2255	crypt_free_req(cc, req: req_of_dmreq(cc, dmreq), base_bio: io->base_bio);
2256
2257	if (!atomic_dec_and_test(v: &ctx->cc_pending))
2258	return;
2259
2260	/*
2261	* The request is fully completed: for inline writes, let
2262	* kcryptd_crypt_write_convert() do the IO submission.
2263	*/
2264	if (bio_data_dir(io->base_bio) == READ) {
2265	kcryptd_crypt_read_done(io);
2266	return;
2267	}
2268
2269	if (kcryptd_crypt_write_inline(cc, ctx)) {
2270	complete(&ctx->restart);
2271	return;
2272	}
2273
2274	kcryptd_crypt_write_io_submit(io, async: 1);
2275	}
2276
2277	static void kcryptd_crypt(struct work_struct *work)
2278	{
2279	struct dm_crypt_io *io = container_of(work, struct dm_crypt_io, work);
2280
2281	if (bio_data_dir(io->base_bio) == READ)
2282	kcryptd_crypt_read_convert(io);
2283	else
2284	kcryptd_crypt_write_convert(io);
2285	}
2286
2287	static void kcryptd_queue_crypt(struct dm_crypt_io *io)
2288	{
2289	struct crypt_config *cc = io->cc;
2290
2291	if ((bio_data_dir(io->base_bio) == READ && test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags)) ||
2292	(bio_data_dir(io->base_bio) == WRITE && test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))) {
2293	/*
2294	* in_hardirq(): Crypto API's skcipher_walk_first() refuses to work in hard IRQ context.
2295	* irqs_disabled(): the kernel may run some IO completion from the idle thread, but
2296	* it is being executed with irqs disabled.
2297	*/
2298	if (in_hardirq() || irqs_disabled()) {
2299	INIT_WORK(&io->work, kcryptd_crypt);
2300	queue_work(wq: system_bh_wq, work: &io->work);
2301	return;
2302	} else {
2303	kcryptd_crypt(work: &io->work);
2304	return;
2305	}
2306	}
2307
2308	INIT_WORK(&io->work, kcryptd_crypt);
2309	queue_work(wq: cc->crypt_queue, work: &io->work);
2310	}
2311
2312	static void crypt_free_tfms_aead(struct crypt_config *cc)
2313	{
2314	if (!cc->cipher_tfm.tfms_aead)
2315	return;
2316
2317	if (cc->cipher_tfm.tfms_aead[0] && !IS_ERR(ptr: cc->cipher_tfm.tfms_aead[0])) {
2318	crypto_free_aead(tfm: cc->cipher_tfm.tfms_aead[0]);
2319	cc->cipher_tfm.tfms_aead[0] = NULL;
2320	}
2321
2322	kfree(objp: cc->cipher_tfm.tfms_aead);
2323	cc->cipher_tfm.tfms_aead = NULL;
2324	}
2325
2326	static void crypt_free_tfms_skcipher(struct crypt_config *cc)
2327	{
2328	unsigned int i;
2329
2330	if (!cc->cipher_tfm.tfms)
2331	return;
2332
2333	for (i = 0; i < cc->tfms_count; i++)
2334	if (cc->cipher_tfm.tfms[i] && !IS_ERR(ptr: cc->cipher_tfm.tfms[i])) {
2335	crypto_free_skcipher(tfm: cc->cipher_tfm.tfms[i]);
2336	cc->cipher_tfm.tfms[i] = NULL;
2337	}
2338
2339	kfree(objp: cc->cipher_tfm.tfms);
2340	cc->cipher_tfm.tfms = NULL;
2341	}
2342
2343	static void crypt_free_tfms(struct crypt_config *cc)
2344	{
2345	if (crypt_integrity_aead(cc))
2346	crypt_free_tfms_aead(cc);
2347	else
2348	crypt_free_tfms_skcipher(cc);
2349	}
2350
2351	static int crypt_alloc_tfms_skcipher(struct crypt_config *cc, char *ciphermode)
2352	{
2353	unsigned int i;
2354	int err;
2355
2356	cc->cipher_tfm.tfms = kcalloc(cc->tfms_count,
2357	sizeof(struct crypto_skcipher *),
2358	GFP_KERNEL);
2359	if (!cc->cipher_tfm.tfms)
2360	return -ENOMEM;
2361
2362	for (i = 0; i < cc->tfms_count; i++) {
2363	cc->cipher_tfm.tfms[i] = crypto_alloc_skcipher(alg_name: ciphermode, type: 0,
2364	CRYPTO_ALG_ALLOCATES_MEMORY);
2365	if (IS_ERR(ptr: cc->cipher_tfm.tfms[i])) {
2366	err = PTR_ERR(ptr: cc->cipher_tfm.tfms[i]);
2367	crypt_free_tfms(cc);
2368	return err;
2369	}
2370	}
2371
2372	/*
2373	* dm-crypt performance can vary greatly depending on which crypto
2374	* algorithm implementation is used. Help people debug performance
2375	* problems by logging the ->cra_driver_name.
2376	*/
2377	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2378	crypto_skcipher_alg(any_tfm(cc))->base.cra_driver_name);
2379	return 0;
2380	}
2381
2382	static int crypt_alloc_tfms_aead(struct crypt_config *cc, char *ciphermode)
2383	{
2384	int err;
2385
2386	cc->cipher_tfm.tfms = kmalloc(sizeof(struct crypto_aead *), GFP_KERNEL);
2387	if (!cc->cipher_tfm.tfms)
2388	return -ENOMEM;
2389
2390	cc->cipher_tfm.tfms_aead[0] = crypto_alloc_aead(alg_name: ciphermode, type: 0,
2391	CRYPTO_ALG_ALLOCATES_MEMORY);
2392	if (IS_ERR(ptr: cc->cipher_tfm.tfms_aead[0])) {
2393	err = PTR_ERR(ptr: cc->cipher_tfm.tfms_aead[0]);
2394	crypt_free_tfms(cc);
2395	return err;
2396	}
2397
2398	DMDEBUG_LIMIT("%s using implementation \"%s\"", ciphermode,
2399	crypto_aead_alg(any_tfm_aead(cc))->base.cra_driver_name);
2400	return 0;
2401	}
2402
2403	static int crypt_alloc_tfms(struct crypt_config *cc, char *ciphermode)
2404	{
2405	if (crypt_integrity_aead(cc))
2406	return crypt_alloc_tfms_aead(cc, ciphermode);
2407	else
2408	return crypt_alloc_tfms_skcipher(cc, ciphermode);
2409	}
2410
2411	static unsigned int crypt_subkey_size(struct crypt_config *cc)
2412	{
2413	return (cc->key_size - cc->key_extra_size) >> ilog2(cc->tfms_count);
2414	}
2415
2416	static unsigned int crypt_authenckey_size(struct crypt_config *cc)
2417	{
2418	return crypt_subkey_size(cc) + RTA_SPACE(sizeof(struct crypto_authenc_key_param));
2419	}
2420
2421	/*
2422	* If AEAD is composed like authenc(hmac(sha256),xts(aes)),
2423	* the key must be for some reason in special format.
2424	* This funcion converts cc->key to this special format.
2425	*/
2426	static void crypt_copy_authenckey(char *p, const void *key,
2427	unsigned int enckeylen, unsigned int authkeylen)
2428	{
2429	struct crypto_authenc_key_param *param;
2430	struct rtattr *rta;
2431
2432	rta = (struct rtattr *)p;
2433	param = RTA_DATA(rta);
2434	param->enckeylen = cpu_to_be32(enckeylen);
2435	rta->rta_len = RTA_LENGTH(sizeof(*param));
2436	rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
2437	p += RTA_SPACE(sizeof(*param));
2438	memcpy(p, key + enckeylen, authkeylen);
2439	p += authkeylen;
2440	memcpy(p, key, enckeylen);
2441	}
2442
2443	static int crypt_setkey(struct crypt_config *cc)
2444	{
2445	unsigned int subkey_size;
2446	int err = 0, i, r;
2447
2448	/* Ignore extra keys (which are used for IV etc) */
2449	subkey_size = crypt_subkey_size(cc);
2450
2451	if (crypt_integrity_hmac(cc)) {
2452	if (subkey_size < cc->key_mac_size)
2453	return -EINVAL;
2454
2455	crypt_copy_authenckey(p: cc->authenc_key, key: cc->key,
2456	enckeylen: subkey_size - cc->key_mac_size,
2457	authkeylen: cc->key_mac_size);
2458	}
2459
2460	for (i = 0; i < cc->tfms_count; i++) {
2461	if (crypt_integrity_hmac(cc))
2462	r = crypto_aead_setkey(tfm: cc->cipher_tfm.tfms_aead[i],
2463	key: cc->authenc_key, keylen: crypt_authenckey_size(cc));
2464	else if (crypt_integrity_aead(cc))
2465	r = crypto_aead_setkey(tfm: cc->cipher_tfm.tfms_aead[i],
2466	key: cc->key + (i * subkey_size),
2467	keylen: subkey_size);
2468	else
2469	r = crypto_skcipher_setkey(tfm: cc->cipher_tfm.tfms[i],
2470	key: cc->key + (i * subkey_size),
2471	keylen: subkey_size);
2472	if (r)
2473	err = r;
2474	}
2475
2476	if (crypt_integrity_hmac(cc))
2477	memzero_explicit(s: cc->authenc_key, count: crypt_authenckey_size(cc));
2478
2479	return err;
2480	}
2481
2482	#ifdef CONFIG_KEYS
2483
2484	static bool contains_whitespace(const char *str)
2485	{
2486	while (*str)
2487	if (isspace(*str++))
2488	return true;
2489	return false;
2490	}
2491
2492	static int set_key_user(struct crypt_config *cc, struct key *key)
2493	{
2494	const struct user_key_payload *ukp;
2495
2496	ukp = user_key_payload_locked(key);
2497	if (!ukp)
2498	return -EKEYREVOKED;
2499
2500	if (cc->key_size != ukp->datalen)
2501	return -EINVAL;
2502
2503	memcpy(cc->key, ukp->data, cc->key_size);
2504
2505	return 0;
2506	}
2507
2508	static int set_key_encrypted(struct crypt_config *cc, struct key *key)
2509	{
2510	const struct encrypted_key_payload *ekp;
2511
2512	ekp = key->payload.data[0];
2513	if (!ekp)
2514	return -EKEYREVOKED;
2515
2516	if (cc->key_size != ekp->decrypted_datalen)
2517	return -EINVAL;
2518
2519	memcpy(cc->key, ekp->decrypted_data, cc->key_size);
2520
2521	return 0;
2522	}
2523
2524	static int set_key_trusted(struct crypt_config *cc, struct key *key)
2525	{
2526	const struct trusted_key_payload *tkp;
2527
2528	tkp = key->payload.data[0];
2529	if (!tkp)
2530	return -EKEYREVOKED;
2531
2532	if (cc->key_size != tkp->key_len)
2533	return -EINVAL;
2534
2535	memcpy(cc->key, tkp->key, cc->key_size);
2536
2537	return 0;
2538	}
2539
2540	static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2541	{
2542	char *new_key_string, *key_desc;
2543	int ret;
2544	struct key_type *type;
2545	struct key *key;
2546	int (*set_key)(struct crypt_config *cc, struct key *key);
2547
2548	/*
2549	* Reject key_string with whitespace. dm core currently lacks code for
2550	* proper whitespace escaping in arguments on DM_TABLE_STATUS path.
2551	*/
2552	if (contains_whitespace(str: key_string)) {
2553	DMERR("whitespace chars not allowed in key string");
2554	return -EINVAL;
2555	}
2556
2557	/* look for next ':' separating key_type from key_description */
2558	key_desc = strchr(key_string, ':');
2559	if (!key_desc || key_desc == key_string || !strlen(key_desc + 1))
2560	return -EINVAL;
2561
2562	if (!strncmp(key_string, "logon:", key_desc - key_string + 1)) {
2563	type = &key_type_logon;
2564	set_key = set_key_user;
2565	} else if (!strncmp(key_string, "user:", key_desc - key_string + 1)) {
2566	type = &key_type_user;
2567	set_key = set_key_user;
2568	} else if (IS_ENABLED(CONFIG_ENCRYPTED_KEYS) &&
2569	!strncmp(key_string, "encrypted:", key_desc - key_string + 1)) {
2570	type = &key_type_encrypted;
2571	set_key = set_key_encrypted;
2572	} else if (IS_ENABLED(CONFIG_TRUSTED_KEYS) &&
2573	!strncmp(key_string, "trusted:", key_desc - key_string + 1)) {
2574	type = &key_type_trusted;
2575	set_key = set_key_trusted;
2576	} else {
2577	return -EINVAL;
2578	}
2579
2580	new_key_string = kstrdup(s: key_string, GFP_KERNEL);
2581	if (!new_key_string)
2582	return -ENOMEM;
2583
2584	key = request_key(type, description: key_desc + 1, NULL);
2585	if (IS_ERR(ptr: key)) {
2586	ret = PTR_ERR(ptr: key);
2587	goto free_new_key_string;
2588	}
2589
2590	down_read(sem: &key->sem);
2591	ret = set_key(cc, key);
2592	up_read(sem: &key->sem);
2593	key_put(key);
2594	if (ret < 0)
2595	goto free_new_key_string;
2596
2597	/* clear the flag since following operations may invalidate previously valid key */
2598	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2599
2600	ret = crypt_setkey(cc);
2601	if (ret)
2602	goto free_new_key_string;
2603
2604	set_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2605	kfree_sensitive(objp: cc->key_string);
2606	cc->key_string = new_key_string;
2607	return 0;
2608
2609	free_new_key_string:
2610	kfree_sensitive(objp: new_key_string);
2611	return ret;
2612	}
2613
2614	static int get_key_size(char **key_string)
2615	{
2616	char *colon, dummy;
2617	int ret;
2618
2619	if (*key_string[0] != ':')
2620	return strlen(*key_string) >> 1;
2621
2622	/* look for next ':' in key string */
2623	colon = strpbrk(*key_string + 1, ":");
2624	if (!colon)
2625	return -EINVAL;
2626
2627	if (sscanf(*key_string + 1, "%u%c", &ret, &dummy) != 2 || dummy != ':')
2628	return -EINVAL;
2629
2630	*key_string = colon;
2631
2632	/* remaining key string should be :<logon|user>:<key_desc> */
2633
2634	return ret;
2635	}
2636
2637	#else
2638
2639	static int crypt_set_keyring_key(struct crypt_config *cc, const char *key_string)
2640	{
2641	return -EINVAL;
2642	}
2643
2644	static int get_key_size(char **key_string)
2645	{
2646	return (*key_string[0] == ':') ? -EINVAL : (int)(strlen(*key_string) >> 1);
2647	}
2648
2649	#endif /* CONFIG_KEYS */
2650
2651	static int crypt_set_key(struct crypt_config *cc, char *key)
2652	{
2653	int r = -EINVAL;
2654	int key_string_len = strlen(key);
2655
2656	/* Hyphen (which gives a key_size of zero) means there is no key. */
2657	if (!cc->key_size && strcmp(key, "-"))
2658	goto out;
2659
2660	/* ':' means the key is in kernel keyring, short-circuit normal key processing */
2661	if (key[0] == ':') {
2662	r = crypt_set_keyring_key(cc, key_string: key + 1);
2663	goto out;
2664	}
2665
2666	/* clear the flag since following operations may invalidate previously valid key */
2667	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2668
2669	/* wipe references to any kernel keyring key */
2670	kfree_sensitive(objp: cc->key_string);
2671	cc->key_string = NULL;
2672
2673	/* Decode key from its hex representation. */
2674	if (cc->key_size && hex2bin(dst: cc->key, src: key, count: cc->key_size) < 0)
2675	goto out;
2676
2677	r = crypt_setkey(cc);
2678	if (!r)
2679	set_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2680
2681	out:
2682	/* Hex key string not needed after here, so wipe it. */
2683	memset(key, '0', key_string_len);
2684
2685	return r;
2686	}
2687
2688	static int crypt_wipe_key(struct crypt_config *cc)
2689	{
2690	int r;
2691
2692	clear_bit(nr: DM_CRYPT_KEY_VALID, addr: &cc->flags);
2693	get_random_bytes(buf: &cc->key, len: cc->key_size);
2694
2695	/* Wipe IV private keys */
2696	if (cc->iv_gen_ops && cc->iv_gen_ops->wipe) {
2697	r = cc->iv_gen_ops->wipe(cc);
2698	if (r)
2699	return r;
2700	}
2701
2702	kfree_sensitive(objp: cc->key_string);
2703	cc->key_string = NULL;
2704	r = crypt_setkey(cc);
2705	memset(&cc->key, 0, cc->key_size * sizeof(u8));
2706
2707	return r;
2708	}
2709
2710	static void crypt_calculate_pages_per_client(void)
2711	{
2712	unsigned long pages = (totalram_pages() - totalhigh_pages()) * DM_CRYPT_MEMORY_PERCENT / 100;
2713
2714	if (!dm_crypt_clients_n)
2715	return;
2716
2717	pages /= dm_crypt_clients_n;
2718	if (pages < DM_CRYPT_MIN_PAGES_PER_CLIENT)
2719	pages = DM_CRYPT_MIN_PAGES_PER_CLIENT;
2720	dm_crypt_pages_per_client = pages;
2721	}
2722
2723	static void *crypt_page_alloc(gfp_t gfp_mask, void *pool_data)
2724	{
2725	struct crypt_config *cc = pool_data;
2726	struct page *page;
2727
2728	/*
2729	* Note, percpu_counter_read_positive() may over (and under) estimate
2730	* the current usage by at most (batch - 1) * num_online_cpus() pages,
2731	* but avoids potential spinlock contention of an exact result.
2732	*/
2733	if (unlikely(percpu_counter_read_positive(&cc->n_allocated_pages) >= dm_crypt_pages_per_client) &&
2734	likely(gfp_mask & __GFP_NORETRY))
2735	return NULL;
2736
2737	page = alloc_page(gfp_mask);
2738	if (likely(page != NULL))
2739	percpu_counter_add(fbc: &cc->n_allocated_pages, amount: 1);
2740
2741	return page;
2742	}
2743
2744	static void crypt_page_free(void *page, void *pool_data)
2745	{
2746	struct crypt_config *cc = pool_data;
2747
2748	__free_page(page);
2749	percpu_counter_sub(fbc: &cc->n_allocated_pages, amount: 1);
2750	}
2751
2752	static void crypt_dtr(struct dm_target *ti)
2753	{
2754	struct crypt_config *cc = ti->private;
2755
2756	ti->private = NULL;
2757
2758	if (!cc)
2759	return;
2760
2761	if (cc->write_thread)
2762	kthread_stop(k: cc->write_thread);
2763
2764	if (cc->io_queue)
2765	destroy_workqueue(wq: cc->io_queue);
2766	if (cc->crypt_queue)
2767	destroy_workqueue(wq: cc->crypt_queue);
2768
2769	if (cc->workqueue_id)
2770	ida_free(&workqueue_ida, id: cc->workqueue_id);
2771
2772	crypt_free_tfms(cc);
2773
2774	bioset_exit(&cc->bs);
2775
2776	mempool_exit(pool: &cc->page_pool);
2777	mempool_exit(pool: &cc->req_pool);
2778	mempool_exit(pool: &cc->tag_pool);
2779
2780	WARN_ON(percpu_counter_sum(&cc->n_allocated_pages) != 0);
2781	percpu_counter_destroy(fbc: &cc->n_allocated_pages);
2782
2783	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
2784	cc->iv_gen_ops->dtr(cc);
2785
2786	if (cc->dev)
2787	dm_put_device(ti, d: cc->dev);
2788
2789	kfree_sensitive(objp: cc->cipher_string);
2790	kfree_sensitive(objp: cc->key_string);
2791	kfree_sensitive(objp: cc->cipher_auth);
2792	kfree_sensitive(objp: cc->authenc_key);
2793
2794	mutex_destroy(lock: &cc->bio_alloc_lock);
2795
2796	/* Must zero key material before freeing */
2797	kfree_sensitive(objp: cc);
2798
2799	spin_lock(lock: &dm_crypt_clients_lock);
2800	WARN_ON(!dm_crypt_clients_n);
2801	dm_crypt_clients_n--;
2802	crypt_calculate_pages_per_client();
2803	spin_unlock(lock: &dm_crypt_clients_lock);
2804
2805	dm_audit_log_dtr(DM_MSG_PREFIX, ti, result: 1);
2806	}
2807
2808	static int crypt_ctr_ivmode(struct dm_target *ti, const char *ivmode)
2809	{
2810	struct crypt_config *cc = ti->private;
2811
2812	if (crypt_integrity_aead(cc))
2813	cc->iv_size = crypto_aead_ivsize(tfm: any_tfm_aead(cc));
2814	else
2815	cc->iv_size = crypto_skcipher_ivsize(tfm: any_tfm(cc));
2816
2817	if (cc->iv_size)
2818	/* at least a 64 bit sector number should fit in our buffer */
2819	cc->iv_size = max(cc->iv_size,
2820	(unsigned int)(sizeof(u64) / sizeof(u8)));
2821	else if (ivmode) {
2822	DMWARN("Selected cipher does not support IVs");
2823	ivmode = NULL;
2824	}
2825
2826	/* Choose ivmode, see comments at iv code. */
2827	if (ivmode == NULL)
2828	cc->iv_gen_ops = NULL;
2829	else if (strcmp(ivmode, "plain") == 0)
2830	cc->iv_gen_ops = &crypt_iv_plain_ops;
2831	else if (strcmp(ivmode, "plain64") == 0)
2832	cc->iv_gen_ops = &crypt_iv_plain64_ops;
2833	else if (strcmp(ivmode, "plain64be") == 0)
2834	cc->iv_gen_ops = &crypt_iv_plain64be_ops;
2835	else if (strcmp(ivmode, "essiv") == 0)
2836	cc->iv_gen_ops = &crypt_iv_essiv_ops;
2837	else if (strcmp(ivmode, "benbi") == 0)
2838	cc->iv_gen_ops = &crypt_iv_benbi_ops;
2839	else if (strcmp(ivmode, "null") == 0)
2840	cc->iv_gen_ops = &crypt_iv_null_ops;
2841	else if (strcmp(ivmode, "eboiv") == 0)
2842	cc->iv_gen_ops = &crypt_iv_eboiv_ops;
2843	else if (strcmp(ivmode, "elephant") == 0) {
2844	cc->iv_gen_ops = &crypt_iv_elephant_ops;
2845	cc->key_parts = 2;
2846	cc->key_extra_size = cc->key_size / 2;
2847	if (cc->key_extra_size > ELEPHANT_MAX_KEY_SIZE)
2848	return -EINVAL;
2849	set_bit(nr: CRYPT_ENCRYPT_PREPROCESS, addr: &cc->cipher_flags);
2850	} else if (strcmp(ivmode, "lmk") == 0) {
2851	cc->iv_gen_ops = &crypt_iv_lmk_ops;
2852	/*
2853	* Version 2 and 3 is recognised according
2854	* to length of provided multi-key string.
2855	* If present (version 3), last key is used as IV seed.
2856	* All keys (including IV seed) are always the same size.
2857	*/
2858	if (cc->key_size % cc->key_parts) {
2859	cc->key_parts++;
2860	cc->key_extra_size = cc->key_size / cc->key_parts;
2861	}
2862	} else if (strcmp(ivmode, "tcw") == 0) {
2863	cc->iv_gen_ops = &crypt_iv_tcw_ops;
2864	cc->key_parts += 2; /* IV + whitening */
2865	cc->key_extra_size = cc->iv_size + TCW_WHITENING_SIZE;
2866	} else if (strcmp(ivmode, "random") == 0) {
2867	cc->iv_gen_ops = &crypt_iv_random_ops;
2868	/* Need storage space in integrity fields. */
2869	cc->integrity_iv_size = cc->iv_size;
2870	} else {
2871	ti->error = "Invalid IV mode";
2872	return -EINVAL;
2873	}
2874
2875	return 0;
2876	}
2877
2878	/*
2879	* Workaround to parse HMAC algorithm from AEAD crypto API spec.
2880	* The HMAC is needed to calculate tag size (HMAC digest size).
2881	* This should be probably done by crypto-api calls (once available...)
2882	*/
2883	static int crypt_ctr_auth_cipher(struct crypt_config *cc, char *cipher_api)
2884	{
2885	char *start, *end, *mac_alg = NULL;
2886	struct crypto_ahash *mac;
2887
2888	if (!strstarts(str: cipher_api, prefix: "authenc("))
2889	return 0;
2890
2891	start = strchr(cipher_api, '(');
2892	end = strchr(cipher_api, ',');
2893	if (!start || !end || ++start > end)
2894	return -EINVAL;
2895
2896	mac_alg = kmemdup_nul(s: start, len: end - start, GFP_KERNEL);
2897	if (!mac_alg)
2898	return -ENOMEM;
2899
2900	mac = crypto_alloc_ahash(alg_name: mac_alg, type: 0, CRYPTO_ALG_ALLOCATES_MEMORY);
2901	kfree(objp: mac_alg);
2902
2903	if (IS_ERR(ptr: mac))
2904	return PTR_ERR(ptr: mac);
2905
2906	if (!test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags))
2907	cc->key_mac_size = crypto_ahash_digestsize(tfm: mac);
2908	crypto_free_ahash(tfm: mac);
2909
2910	cc->authenc_key = kmalloc(crypt_authenckey_size(cc), GFP_KERNEL);
2911	if (!cc->authenc_key)
2912	return -ENOMEM;
2913
2914	return 0;
2915	}
2916
2917	static int crypt_ctr_cipher_new(struct dm_target *ti, char *cipher_in, char *key,
2918	char **ivmode, char **ivopts)
2919	{
2920	struct crypt_config *cc = ti->private;
2921	char *tmp, *cipher_api, buf[CRYPTO_MAX_ALG_NAME];
2922	int ret = -EINVAL;
2923
2924	cc->tfms_count = 1;
2925
2926	/*
2927	* New format (capi: prefix)
2928	* capi:cipher_api_spec-iv:ivopts
2929	*/
2930	tmp = &cipher_in[strlen("capi:")];
2931
2932	/* Separate IV options if present, it can contain another '-' in hash name */
2933	*ivopts = strrchr(tmp, ':');
2934	if (*ivopts) {
2935	**ivopts = '\0';
2936	(*ivopts)++;
2937	}
2938	/* Parse IV mode */
2939	*ivmode = strrchr(tmp, '-');
2940	if (*ivmode) {
2941	**ivmode = '\0';
2942	(*ivmode)++;
2943	}
2944	/* The rest is crypto API spec */
2945	cipher_api = tmp;
2946
2947	/* Alloc AEAD, can be used only in new format. */
2948	if (crypt_integrity_aead(cc)) {
2949	ret = crypt_ctr_auth_cipher(cc, cipher_api);
2950	if (ret < 0) {
2951	ti->error = "Invalid AEAD cipher spec";
2952	return ret;
2953	}
2954	}
2955
2956	if (*ivmode && !strcmp(*ivmode, "lmk"))
2957	cc->tfms_count = 64;
2958
2959	if (*ivmode && !strcmp(*ivmode, "essiv")) {
2960	if (!*ivopts) {
2961	ti->error = "Digest algorithm missing for ESSIV mode";
2962	return -EINVAL;
2963	}
2964	ret = snprintf(buf, CRYPTO_MAX_ALG_NAME, fmt: "essiv(%s,%s)",
2965	cipher_api, *ivopts);
2966	if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
2967	ti->error = "Cannot allocate cipher string";
2968	return -ENOMEM;
2969	}
2970	cipher_api = buf;
2971	}
2972
2973	cc->key_parts = cc->tfms_count;
2974
2975	/* Allocate cipher */
2976	ret = crypt_alloc_tfms(cc, ciphermode: cipher_api);
2977	if (ret < 0) {
2978	ti->error = "Error allocating crypto tfm";
2979	return ret;
2980	}
2981
2982	if (crypt_integrity_aead(cc))
2983	cc->iv_size = crypto_aead_ivsize(tfm: any_tfm_aead(cc));
2984	else
2985	cc->iv_size = crypto_skcipher_ivsize(tfm: any_tfm(cc));
2986
2987	return 0;
2988	}
2989
2990	static int crypt_ctr_cipher_old(struct dm_target *ti, char *cipher_in, char *key,
2991	char **ivmode, char **ivopts)
2992	{
2993	struct crypt_config *cc = ti->private;
2994	char *tmp, *cipher, *chainmode, *keycount;
2995	char *cipher_api = NULL;
2996	int ret = -EINVAL;
2997	char dummy;
2998
2999	if (strchr(cipher_in, '(') || crypt_integrity_aead(cc)) {
3000	ti->error = "Bad cipher specification";
3001	return -EINVAL;
3002	}
3003
3004	/*
3005	* Legacy dm-crypt cipher specification
3006	* cipher[:keycount]-mode-iv:ivopts
3007	*/
3008	tmp = cipher_in;
3009	keycount = strsep(&tmp, "-");
3010	cipher = strsep(&keycount, ":");
3011
3012	if (!keycount)
3013	cc->tfms_count = 1;
3014	else if (sscanf(keycount, "%u%c", &cc->tfms_count, &dummy) != 1 ||
3015	!is_power_of_2(n: cc->tfms_count)) {
3016	ti->error = "Bad cipher key count specification";
3017	return -EINVAL;
3018	}
3019	cc->key_parts = cc->tfms_count;
3020
3021	chainmode = strsep(&tmp, "-");
3022	*ivmode = strsep(&tmp, ":");
3023	*ivopts = tmp;
3024
3025	/*
3026	* For compatibility with the original dm-crypt mapping format, if
3027	* only the cipher name is supplied, use cbc-plain.
3028	*/
3029	if (!chainmode || (!strcmp(chainmode, "plain") && !*ivmode)) {
3030	chainmode = "cbc";
3031	*ivmode = "plain";
3032	}
3033
3034	if (strcmp(chainmode, "ecb") && !*ivmode) {
3035	ti->error = "IV mechanism required";
3036	return -EINVAL;
3037	}
3038
3039	cipher_api = kmalloc(CRYPTO_MAX_ALG_NAME, GFP_KERNEL);
3040	if (!cipher_api)
3041	goto bad_mem;
3042
3043	if (*ivmode && !strcmp(*ivmode, "essiv")) {
3044	if (!*ivopts) {
3045	ti->error = "Digest algorithm missing for ESSIV mode";
3046	kfree(objp: cipher_api);
3047	return -EINVAL;
3048	}
3049	ret = snprintf(buf: cipher_api, CRYPTO_MAX_ALG_NAME,
3050	fmt: "essiv(%s(%s),%s)", chainmode, cipher, *ivopts);
3051	} else {
3052	ret = snprintf(buf: cipher_api, CRYPTO_MAX_ALG_NAME,
3053	fmt: "%s(%s)", chainmode, cipher);
3054	}
3055	if (ret < 0 || ret >= CRYPTO_MAX_ALG_NAME) {
3056	kfree(objp: cipher_api);
3057	goto bad_mem;
3058	}
3059
3060	/* Allocate cipher */
3061	ret = crypt_alloc_tfms(cc, ciphermode: cipher_api);
3062	if (ret < 0) {
3063	ti->error = "Error allocating crypto tfm";
3064	kfree(objp: cipher_api);
3065	return ret;
3066	}
3067	kfree(objp: cipher_api);
3068
3069	return 0;
3070	bad_mem:
3071	ti->error = "Cannot allocate cipher strings";
3072	return -ENOMEM;
3073	}
3074
3075	static int crypt_ctr_cipher(struct dm_target *ti, char *cipher_in, char *key)
3076	{
3077	struct crypt_config *cc = ti->private;
3078	char *ivmode = NULL, *ivopts = NULL;
3079	int ret;
3080
3081	cc->cipher_string = kstrdup(s: cipher_in, GFP_KERNEL);
3082	if (!cc->cipher_string) {
3083	ti->error = "Cannot allocate cipher strings";
3084	return -ENOMEM;
3085	}
3086
3087	if (strstarts(str: cipher_in, prefix: "capi:"))
3088	ret = crypt_ctr_cipher_new(ti, cipher_in, key, ivmode: &ivmode, ivopts: &ivopts);
3089	else
3090	ret = crypt_ctr_cipher_old(ti, cipher_in, key, ivmode: &ivmode, ivopts: &ivopts);
3091	if (ret)
3092	return ret;
3093
3094	/* Initialize IV */
3095	ret = crypt_ctr_ivmode(ti, ivmode);
3096	if (ret < 0)
3097	return ret;
3098
3099	/* Initialize and set key */
3100	ret = crypt_set_key(cc, key);
3101	if (ret < 0) {
3102	ti->error = "Error decoding and setting key";
3103	return ret;
3104	}
3105
3106	/* Allocate IV */
3107	if (cc->iv_gen_ops && cc->iv_gen_ops->ctr) {
3108	ret = cc->iv_gen_ops->ctr(cc, ti, ivopts);
3109	if (ret < 0) {
3110	ti->error = "Error creating IV";
3111	return ret;
3112	}
3113	}
3114
3115	/* Initialize IV (set keys for ESSIV etc) */
3116	if (cc->iv_gen_ops && cc->iv_gen_ops->init) {
3117	ret = cc->iv_gen_ops->init(cc);
3118	if (ret < 0) {
3119	ti->error = "Error initialising IV";
3120	return ret;
3121	}
3122	}
3123
3124	/* wipe the kernel key payload copy */
3125	if (cc->key_string)
3126	memset(cc->key, 0, cc->key_size * sizeof(u8));
3127
3128	return ret;
3129	}
3130
3131	static int crypt_ctr_optional(struct dm_target *ti, unsigned int argc, char **argv)
3132	{
3133	struct crypt_config *cc = ti->private;
3134	struct dm_arg_set as;
3135	static const struct dm_arg _args[] = {
3136	{0, 9, "Invalid number of feature args"},
3137	};
3138	unsigned int opt_params, val;
3139	const char *opt_string, *sval;
3140	char dummy;
3141	int ret;
3142
3143	/* Optional parameters */
3144	as.argc = argc;
3145	as.argv = argv;
3146
3147	ret = dm_read_arg_group(arg: _args, arg_set: &as, num_args: &opt_params, error: &ti->error);
3148	if (ret)
3149	return ret;
3150
3151	while (opt_params--) {
3152	opt_string = dm_shift_arg(as: &as);
3153	if (!opt_string) {
3154	ti->error = "Not enough feature arguments";
3155	return -EINVAL;
3156	}
3157
3158	if (!strcasecmp(s1: opt_string, s2: "allow_discards"))
3159	ti->num_discard_bios = 1;
3160
3161	else if (!strcasecmp(s1: opt_string, s2: "same_cpu_crypt"))
3162	set_bit(nr: DM_CRYPT_SAME_CPU, addr: &cc->flags);
3163	else if (!strcasecmp(s1: opt_string, s2: "high_priority"))
3164	set_bit(nr: DM_CRYPT_HIGH_PRIORITY, addr: &cc->flags);
3165
3166	else if (!strcasecmp(s1: opt_string, s2: "submit_from_crypt_cpus"))
3167	set_bit(nr: DM_CRYPT_NO_OFFLOAD, addr: &cc->flags);
3168	else if (!strcasecmp(s1: opt_string, s2: "no_read_workqueue"))
3169	set_bit(nr: DM_CRYPT_NO_READ_WORKQUEUE, addr: &cc->flags);
3170	else if (!strcasecmp(s1: opt_string, s2: "no_write_workqueue"))
3171	set_bit(nr: DM_CRYPT_NO_WRITE_WORKQUEUE, addr: &cc->flags);
3172	else if (sscanf(opt_string, "integrity:%u:", &val) == 1) {
3173	if (val == 0 || val > MAX_TAG_SIZE) {
3174	ti->error = "Invalid integrity arguments";
3175	return -EINVAL;
3176	}
3177	cc->used_tag_size = val;
3178	sval = strchr(opt_string + strlen("integrity:"), ':') + 1;
3179	if (!strcasecmp(s1: sval, s2: "aead")) {
3180	set_bit(nr: CRYPT_MODE_INTEGRITY_AEAD, addr: &cc->cipher_flags);
3181	} else if (strcasecmp(s1: sval, s2: "none")) {
3182	ti->error = "Unknown integrity profile";
3183	return -EINVAL;
3184	}
3185
3186	cc->cipher_auth = kstrdup(s: sval, GFP_KERNEL);
3187	if (!cc->cipher_auth)
3188	return -ENOMEM;
3189	} else if (sscanf(opt_string, "integrity_key_size:%u%c", &val, &dummy) == 1) {
3190	if (!val) {
3191	ti->error = "Invalid integrity_key_size argument";
3192	return -EINVAL;
3193	}
3194	cc->key_mac_size = val;
3195	set_bit(nr: CRYPT_KEY_MAC_SIZE_SET, addr: &cc->cipher_flags);
3196	} else if (sscanf(opt_string, "sector_size:%hu%c", &cc->sector_size, &dummy) == 1) {
3197	if (cc->sector_size < (1 << SECTOR_SHIFT) ||
3198	cc->sector_size > 4096 ||
3199	(cc->sector_size & (cc->sector_size - 1))) {
3200	ti->error = "Invalid feature value for sector_size";
3201	return -EINVAL;
3202	}
3203	if (ti->len & ((cc->sector_size >> SECTOR_SHIFT) - 1)) {
3204	ti->error = "Device size is not multiple of sector_size feature";
3205	return -EINVAL;
3206	}
3207	cc->sector_shift = __ffs(cc->sector_size) - SECTOR_SHIFT;
3208	} else if (!strcasecmp(s1: opt_string, s2: "iv_large_sectors"))
3209	set_bit(nr: CRYPT_IV_LARGE_SECTORS, addr: &cc->cipher_flags);
3210	else {
3211	ti->error = "Invalid feature arguments";
3212	return -EINVAL;
3213	}
3214	}
3215
3216	return 0;
3217	}
3218
3219	#ifdef CONFIG_BLK_DEV_ZONED
3220	static int crypt_report_zones(struct dm_target *ti,
3221	struct dm_report_zones_args *args, unsigned int nr_zones)
3222	{
3223	struct crypt_config *cc = ti->private;
3224
3225	return dm_report_zones(bdev: cc->dev->bdev, start: cc->start,
3226	sector: cc->start + dm_target_offset(ti, args->next_sector),
3227	args, nr_zones);
3228	}
3229	#else
3230	#define crypt_report_zones NULL
3231	#endif
3232
3233	/*
3234	* Construct an encryption mapping:
3235	* <cipher> [<key>|:<key_size>:<user|logon>:<key_description>] <iv_offset> <dev_path> <start>
3236	*/
3237	static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
3238	{
3239	struct crypt_config *cc;
3240	const char *devname = dm_table_device_name(t: ti->table);
3241	int key_size, wq_id;
3242	unsigned int align_mask;
3243	unsigned int common_wq_flags;
3244	unsigned long long tmpll;
3245	int ret;
3246	size_t iv_size_padding, additional_req_size;
3247	char dummy;
3248
3249	if (argc < 5) {
3250	ti->error = "Not enough arguments";
3251	return -EINVAL;
3252	}
3253
3254	key_size = get_key_size(key_string: &argv[1]);
3255	if (key_size < 0) {
3256	ti->error = "Cannot parse key size";
3257	return -EINVAL;
3258	}
3259
3260	cc = kzalloc(struct_size(cc, key, key_size), GFP_KERNEL);
3261	if (!cc) {
3262	ti->error = "Cannot allocate encryption context";
3263	return -ENOMEM;
3264	}
3265	cc->key_size = key_size;
3266	cc->sector_size = (1 << SECTOR_SHIFT);
3267	cc->sector_shift = 0;
3268
3269	ti->private = cc;
3270
3271	spin_lock(lock: &dm_crypt_clients_lock);
3272	dm_crypt_clients_n++;
3273	crypt_calculate_pages_per_client();
3274	spin_unlock(lock: &dm_crypt_clients_lock);
3275
3276	ret = percpu_counter_init(&cc->n_allocated_pages, 0, GFP_KERNEL);
3277	if (ret < 0)
3278	goto bad;
3279
3280	/* Optional parameters need to be read before cipher constructor */
3281	if (argc > 5) {
3282	ret = crypt_ctr_optional(ti, argc: argc - 5, argv: &argv[5]);
3283	if (ret)
3284	goto bad;
3285	}
3286
3287	ret = crypt_ctr_cipher(ti, cipher_in: argv[0], key: argv[1]);
3288	if (ret < 0)
3289	goto bad;
3290
3291	if (crypt_integrity_aead(cc)) {
3292	cc->dmreq_start = sizeof(struct aead_request);
3293	cc->dmreq_start += crypto_aead_reqsize(tfm: any_tfm_aead(cc));
3294	align_mask = crypto_aead_alignmask(tfm: any_tfm_aead(cc));
3295	} else {
3296	cc->dmreq_start = sizeof(struct skcipher_request);
3297	cc->dmreq_start += crypto_skcipher_reqsize(tfm: any_tfm(cc));
3298	align_mask = crypto_skcipher_alignmask(tfm: any_tfm(cc));
3299	}
3300	cc->dmreq_start = ALIGN(cc->dmreq_start, __alignof__(struct dm_crypt_request));
3301
3302	if (align_mask < CRYPTO_MINALIGN) {
3303	/* Allocate the padding exactly */
3304	iv_size_padding = -(cc->dmreq_start + sizeof(struct dm_crypt_request))
3305	& align_mask;
3306	} else {
3307	/*
3308	* If the cipher requires greater alignment than kmalloc
3309	* alignment, we don't know the exact position of the
3310	* initialization vector. We must assume worst case.
3311	*/
3312	iv_size_padding = align_mask;
3313	}
3314
3315	/* ...| IV + padding | original IV | original sec. number | bio tag offset | */
3316	additional_req_size = sizeof(struct dm_crypt_request) +
3317	iv_size_padding + cc->iv_size +
3318	cc->iv_size +
3319	sizeof(uint64_t) +
3320	sizeof(unsigned int);
3321
3322	ret = mempool_init_kmalloc_pool(&cc->req_pool, MIN_IOS, cc->dmreq_start + additional_req_size);
3323	if (ret) {
3324	ti->error = "Cannot allocate crypt request mempool";
3325	goto bad;
3326	}
3327
3328	cc->per_bio_data_size = ti->per_io_data_size =
3329	ALIGN(sizeof(struct dm_crypt_io) + cc->dmreq_start + additional_req_size,
3330	ARCH_DMA_MINALIGN);
3331
3332	ret = mempool_init(&cc->page_pool, BIO_MAX_VECS, crypt_page_alloc, crypt_page_free, cc);
3333	if (ret) {
3334	ti->error = "Cannot allocate page mempool";
3335	goto bad;
3336	}
3337
3338	ret = bioset_init(&cc->bs, MIN_IOS, 0, flags: BIOSET_NEED_BVECS);
3339	if (ret) {
3340	ti->error = "Cannot allocate crypt bioset";
3341	goto bad;
3342	}
3343
3344	mutex_init(&cc->bio_alloc_lock);
3345
3346	ret = -EINVAL;
3347	if ((sscanf(argv[2], "%llu%c", &tmpll, &dummy) != 1) ||
3348	(tmpll & ((cc->sector_size >> SECTOR_SHIFT) - 1))) {
3349	ti->error = "Invalid iv_offset sector";
3350	goto bad;
3351	}
3352	cc->iv_offset = tmpll;
3353
3354	ret = dm_get_device(ti, path: argv[3], mode: dm_table_get_mode(t: ti->table), result: &cc->dev);
3355	if (ret) {
3356	ti->error = "Device lookup failed";
3357	goto bad;
3358	}
3359
3360	ret = -EINVAL;
3361	if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1 || tmpll != (sector_t)tmpll) {
3362	ti->error = "Invalid device sector";
3363	goto bad;
3364	}
3365	cc->start = tmpll;
3366
3367	if (bdev_is_zoned(bdev: cc->dev->bdev)) {
3368	/*
3369	* For zoned block devices, we need to preserve the issuer write
3370	* ordering. To do so, disable write workqueues and force inline
3371	* encryption completion.
3372	*/
3373	set_bit(nr: DM_CRYPT_NO_WRITE_WORKQUEUE, addr: &cc->flags);
3374	set_bit(nr: DM_CRYPT_WRITE_INLINE, addr: &cc->flags);
3375
3376	/*
3377	* All zone append writes to a zone of a zoned block device will
3378	* have the same BIO sector, the start of the zone. When the
3379	* cypher IV mode uses sector values, all data targeting a
3380	* zone will be encrypted using the first sector numbers of the
3381	* zone. This will not result in write errors but will
3382	* cause most reads to fail as reads will use the sector values
3383	* for the actual data locations, resulting in IV mismatch.
3384	* To avoid this problem, ask DM core to emulate zone append
3385	* operations with regular writes.
3386	*/
3387	DMDEBUG("Zone append operations will be emulated");
3388	ti->emulate_zone_append = true;
3389	}
3390
3391	if (crypt_integrity_aead(cc) || cc->integrity_iv_size) {
3392	ret = crypt_integrity_ctr(cc, ti);
3393	if (ret)
3394	goto bad;
3395
3396	cc->tag_pool_max_sectors = POOL_ENTRY_SIZE / cc->tuple_size;
3397	if (!cc->tag_pool_max_sectors)
3398	cc->tag_pool_max_sectors = 1;
3399
3400	ret = mempool_init_kmalloc_pool(&cc->tag_pool, MIN_IOS,
3401	cc->tag_pool_max_sectors * cc->tuple_size);
3402	if (ret) {
3403	ti->error = "Cannot allocate integrity tags mempool";
3404	goto bad;
3405	}
3406
3407	cc->tag_pool_max_sectors <<= cc->sector_shift;
3408	}
3409
3410	wq_id = ida_alloc_min(ida: &workqueue_ida, min: 1, GFP_KERNEL);
3411	if (wq_id < 0) {
3412	ti->error = "Couldn't get workqueue id";
3413	ret = wq_id;
3414	goto bad;
3415	}
3416	cc->workqueue_id = wq_id;
3417
3418	ret = -ENOMEM;
3419	common_wq_flags = WQ_MEM_RECLAIM | WQ_SYSFS;
3420	if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3421	common_wq_flags |= WQ_HIGHPRI;
3422
3423	cc->io_queue = alloc_workqueue(fmt: "kcryptd_io-%s-%d", flags: common_wq_flags, max_active: 1, devname, wq_id);
3424	if (!cc->io_queue) {
3425	ti->error = "Couldn't create kcryptd io queue";
3426	goto bad;
3427	}
3428
3429	if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags)) {
3430	cc->crypt_queue = alloc_workqueue(fmt: "kcryptd-%s-%d",
3431	flags: common_wq_flags | WQ_CPU_INTENSIVE,
3432	max_active: 1, devname, wq_id);
3433	} else {
3434	/*
3435	* While crypt_queue is certainly CPU intensive, the use of
3436	* WQ_CPU_INTENSIVE is meaningless with WQ_UNBOUND.
3437	*/
3438	cc->crypt_queue = alloc_workqueue(fmt: "kcryptd-%s-%d",
3439	flags: common_wq_flags | WQ_UNBOUND,
3440	max_active: num_online_cpus(), devname, wq_id);
3441	}
3442	if (!cc->crypt_queue) {
3443	ti->error = "Couldn't create kcryptd queue";
3444	goto bad;
3445	}
3446
3447	spin_lock_init(&cc->write_thread_lock);
3448	cc->write_tree = RB_ROOT;
3449
3450	cc->write_thread = kthread_run(dmcrypt_write, cc, "dmcrypt_write/%s", devname);
3451	if (IS_ERR(ptr: cc->write_thread)) {
3452	ret = PTR_ERR(ptr: cc->write_thread);
3453	cc->write_thread = NULL;
3454	ti->error = "Couldn't spawn write thread";
3455	goto bad;
3456	}
3457	if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3458	set_user_nice(p: cc->write_thread, MIN_NICE);
3459
3460	ti->num_flush_bios = 1;
3461	ti->limit_swap_bios = true;
3462	ti->accounts_remapped_io = true;
3463
3464	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 1);
3465	return 0;
3466
3467	bad:
3468	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 0);
3469	crypt_dtr(ti);
3470	return ret;
3471	}
3472
3473	static int crypt_map(struct dm_target *ti, struct bio *bio)
3474	{
3475	struct dm_crypt_io *io;
3476	struct crypt_config *cc = ti->private;
3477	unsigned max_sectors;
3478
3479	/*
3480	* If bio is REQ_PREFLUSH or REQ_OP_DISCARD, just bypass crypt queues.
3481	* - for REQ_PREFLUSH device-mapper core ensures that no IO is in-flight
3482	* - for REQ_OP_DISCARD caller must use flush if IO ordering matters
3483	*/
3484	if (unlikely(bio->bi_opf & REQ_PREFLUSH ||
3485	bio_op(bio) == REQ_OP_DISCARD)) {
3486	bio_set_dev(bio, bdev: cc->dev->bdev);
3487	if (bio_sectors(bio))
3488	bio->bi_iter.bi_sector = cc->start +
3489	dm_target_offset(ti, bio->bi_iter.bi_sector);
3490	return DM_MAPIO_REMAPPED;
3491	}
3492
3493	/*
3494	* Check if bio is too large, split as needed.
3495	*/
3496	max_sectors = get_max_request_size(cc, bio_data_dir(bio) == WRITE);
3497	if (unlikely(bio_sectors(bio) > max_sectors))
3498	dm_accept_partial_bio(bio, n_sectors: max_sectors);
3499
3500	/*
3501	* Ensure that bio is a multiple of internal sector encryption size
3502	* and is aligned to this size as defined in IO hints.
3503	*/
3504	if (unlikely((bio->bi_iter.bi_sector & ((cc->sector_size >> SECTOR_SHIFT) - 1)) != 0))
3505	return DM_MAPIO_KILL;
3506
3507	if (unlikely(bio->bi_iter.bi_size & (cc->sector_size - 1)))
3508	return DM_MAPIO_KILL;
3509
3510	io = dm_per_bio_data(bio, data_size: cc->per_bio_data_size);
3511	crypt_io_init(io, cc, bio, dm_target_offset(ti, bio->bi_iter.bi_sector));
3512
3513	if (cc->tuple_size) {
3514	unsigned int tag_len = cc->tuple_size * (bio_sectors(bio) >> cc->sector_shift);
3515
3516	if (unlikely(tag_len > KMALLOC_MAX_SIZE))
3517	io->integrity_metadata = NULL;
3518	else
3519	io->integrity_metadata = kmalloc(tag_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
3520
3521	if (unlikely(!io->integrity_metadata)) {
3522	if (bio_sectors(bio) > cc->tag_pool_max_sectors)
3523	dm_accept_partial_bio(bio, n_sectors: cc->tag_pool_max_sectors);
3524	io->integrity_metadata = mempool_alloc(&cc->tag_pool, GFP_NOIO);
3525	io->integrity_metadata_from_pool = true;
3526	}
3527	}
3528
3529	if (crypt_integrity_aead(cc))
3530	io->ctx.r.req_aead = (struct aead_request *)(io + 1);
3531	else
3532	io->ctx.r.req = (struct skcipher_request *)(io + 1);
3533
3534	if (bio_data_dir(io->base_bio) == READ) {
3535	if (kcryptd_io_read(io, CRYPT_MAP_READ_GFP))
3536	kcryptd_queue_read(io);
3537	} else
3538	kcryptd_queue_crypt(io);
3539
3540	return DM_MAPIO_SUBMITTED;
3541	}
3542
3543	static char hex2asc(unsigned char c)
3544	{
3545	return c + '0' + ((unsigned int)(9 - c) >> 4 & 0x27);
3546	}
3547
3548	static void crypt_status(struct dm_target *ti, status_type_t type,
3549	unsigned int status_flags, char *result, unsigned int maxlen)
3550	{
3551	struct crypt_config *cc = ti->private;
3552	unsigned int i, sz = 0;
3553	int num_feature_args = 0;
3554
3555	switch (type) {
3556	case STATUSTYPE_INFO:
3557	result[0] = '\0';
3558	break;
3559
3560	case STATUSTYPE_TABLE:
3561	DMEMIT("%s ", cc->cipher_string);
3562
3563	if (cc->key_size > 0) {
3564	if (cc->key_string)
3565	DMEMIT(":%u:%s", cc->key_size, cc->key_string);
3566	else {
3567	for (i = 0; i < cc->key_size; i++) {
3568	DMEMIT("%c%c", hex2asc(cc->key[i] >> 4),
3569	hex2asc(cc->key[i] & 0xf));
3570	}
3571	}
3572	} else
3573	DMEMIT("-");
3574
3575	DMEMIT(" %llu %s %llu", (unsigned long long)cc->iv_offset,
3576	cc->dev->name, (unsigned long long)cc->start);
3577
3578	num_feature_args += !!ti->num_discard_bios;
3579	num_feature_args += test_bit(DM_CRYPT_SAME_CPU, &cc->flags);
3580	num_feature_args += test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags);
3581	num_feature_args += test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags);
3582	num_feature_args += test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags);
3583	num_feature_args += test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags);
3584	num_feature_args += !!cc->used_tag_size;
3585	num_feature_args += cc->sector_size != (1 << SECTOR_SHIFT);
3586	num_feature_args += test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags);
3587	num_feature_args += test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags);
3588	if (num_feature_args) {
3589	DMEMIT(" %d", num_feature_args);
3590	if (ti->num_discard_bios)
3591	DMEMIT(" allow_discards");
3592	if (test_bit(DM_CRYPT_SAME_CPU, &cc->flags))
3593	DMEMIT(" same_cpu_crypt");
3594	if (test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags))
3595	DMEMIT(" high_priority");
3596	if (test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags))
3597	DMEMIT(" submit_from_crypt_cpus");
3598	if (test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags))
3599	DMEMIT(" no_read_workqueue");
3600	if (test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags))
3601	DMEMIT(" no_write_workqueue");
3602	if (cc->used_tag_size)
3603	DMEMIT(" integrity:%u:%s", cc->used_tag_size, cc->cipher_auth);
3604	if (cc->sector_size != (1 << SECTOR_SHIFT))
3605	DMEMIT(" sector_size:%d", cc->sector_size);
3606	if (test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags))
3607	DMEMIT(" iv_large_sectors");
3608	if (test_bit(CRYPT_KEY_MAC_SIZE_SET, &cc->cipher_flags))
3609	DMEMIT(" integrity_key_size:%u", cc->key_mac_size);
3610	}
3611	break;
3612
3613	case STATUSTYPE_IMA:
3614	DMEMIT_TARGET_NAME_VERSION(ti->type);
3615	DMEMIT(",allow_discards=%c", ti->num_discard_bios ? 'y' : 'n');
3616	DMEMIT(",same_cpu_crypt=%c", test_bit(DM_CRYPT_SAME_CPU, &cc->flags) ? 'y' : 'n');
3617	DMEMIT(",high_priority=%c", test_bit(DM_CRYPT_HIGH_PRIORITY, &cc->flags) ? 'y' : 'n');
3618	DMEMIT(",submit_from_crypt_cpus=%c", test_bit(DM_CRYPT_NO_OFFLOAD, &cc->flags) ?
3619	'y' : 'n');
3620	DMEMIT(",no_read_workqueue=%c", test_bit(DM_CRYPT_NO_READ_WORKQUEUE, &cc->flags) ?
3621	'y' : 'n');
3622	DMEMIT(",no_write_workqueue=%c", test_bit(DM_CRYPT_NO_WRITE_WORKQUEUE, &cc->flags) ?
3623	'y' : 'n');
3624	DMEMIT(",iv_large_sectors=%c", test_bit(CRYPT_IV_LARGE_SECTORS, &cc->cipher_flags) ?
3625	'y' : 'n');
3626
3627	if (cc->used_tag_size)
3628	DMEMIT(",integrity_tag_size=%u,cipher_auth=%s",
3629	cc->used_tag_size, cc->cipher_auth);
3630	if (cc->sector_size != (1 << SECTOR_SHIFT))
3631	DMEMIT(",sector_size=%d", cc->sector_size);
3632	if (cc->cipher_string)
3633	DMEMIT(",cipher_string=%s", cc->cipher_string);
3634
3635	DMEMIT(",key_size=%u", cc->key_size);
3636	DMEMIT(",key_parts=%u", cc->key_parts);
3637	DMEMIT(",key_extra_size=%u", cc->key_extra_size);
3638	DMEMIT(",key_mac_size=%u", cc->key_mac_size);
3639	DMEMIT(";");
3640	break;
3641	}
3642	}
3643
3644	static void crypt_postsuspend(struct dm_target *ti)
3645	{
3646	struct crypt_config *cc = ti->private;
3647
3648	set_bit(nr: DM_CRYPT_SUSPENDED, addr: &cc->flags);
3649	}
3650
3651	static int crypt_preresume(struct dm_target *ti)
3652	{
3653	struct crypt_config *cc = ti->private;
3654
3655	if (!test_bit(DM_CRYPT_KEY_VALID, &cc->flags)) {
3656	DMERR("aborting resume - crypt key is not set.");
3657	return -EAGAIN;
3658	}
3659
3660	return 0;
3661	}
3662
3663	static void crypt_resume(struct dm_target *ti)
3664	{
3665	struct crypt_config *cc = ti->private;
3666
3667	clear_bit(nr: DM_CRYPT_SUSPENDED, addr: &cc->flags);
3668	}
3669
3670	/* Message interface
3671	* key set <key>
3672	* key wipe
3673	*/
3674	static int crypt_message(struct dm_target *ti, unsigned int argc, char **argv,
3675	char *result, unsigned int maxlen)
3676	{
3677	struct crypt_config *cc = ti->private;
3678	int key_size, ret = -EINVAL;
3679
3680	if (argc < 2)
3681	goto error;
3682
3683	if (!strcasecmp(s1: argv[0], s2: "key")) {
3684	if (!test_bit(DM_CRYPT_SUSPENDED, &cc->flags)) {
3685	DMWARN("not suspended during key manipulation.");
3686	return -EINVAL;
3687	}
3688	if (argc == 3 && !strcasecmp(s1: argv[1], s2: "set")) {
3689	/* The key size may not be changed. */
3690	key_size = get_key_size(key_string: &argv[2]);
3691	if (key_size < 0 || cc->key_size != key_size) {
3692	memset(argv[2], '0', strlen(argv[2]));
3693	return -EINVAL;
3694	}
3695
3696	ret = crypt_set_key(cc, key: argv[2]);
3697	if (ret)
3698	return ret;
3699	if (cc->iv_gen_ops && cc->iv_gen_ops->init)
3700	ret = cc->iv_gen_ops->init(cc);
3701	/* wipe the kernel key payload copy */
3702	if (cc->key_string)
3703	memset(cc->key, 0, cc->key_size * sizeof(u8));
3704	return ret;
3705	}
3706	if (argc == 2 && !strcasecmp(s1: argv[1], s2: "wipe"))
3707	return crypt_wipe_key(cc);
3708	}
3709
3710	error:
3711	DMWARN("unrecognised message received.");
3712	return -EINVAL;
3713	}
3714
3715	static int crypt_iterate_devices(struct dm_target *ti,
3716	iterate_devices_callout_fn fn, void *data)
3717	{
3718	struct crypt_config *cc = ti->private;
3719
3720	return fn(ti, cc->dev, cc->start, ti->len, data);
3721	}
3722
3723	static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits)
3724	{
3725	struct crypt_config *cc = ti->private;
3726
3727	limits->logical_block_size =
3728	max_t(unsigned int, limits->logical_block_size, cc->sector_size);
3729	limits->physical_block_size =
3730	max_t(unsigned int, limits->physical_block_size, cc->sector_size);
3731	limits->io_min = max_t(unsigned int, limits->io_min, cc->sector_size);
3732	limits->dma_alignment = limits->logical_block_size - 1;
3733	}
3734
3735	static struct target_type crypt_target = {
3736	.name = "crypt",
3737	.version = {1, 28, 0},
3738	.module = THIS_MODULE,
3739	.ctr = crypt_ctr,
3740	.dtr = crypt_dtr,
3741	.features = DM_TARGET_ZONED_HM,
3742	.report_zones = crypt_report_zones,
3743	.map = crypt_map,
3744	.status = crypt_status,
3745	.postsuspend = crypt_postsuspend,
3746	.preresume = crypt_preresume,
3747	.resume = crypt_resume,
3748	.message = crypt_message,
3749	.iterate_devices = crypt_iterate_devices,
3750	.io_hints = crypt_io_hints,
3751	};
3752	module_dm(crypt);
3753
3754	MODULE_AUTHOR("Jana Saout <jana@saout.de>");
3755	MODULE_DESCRIPTION(DM_NAME " target for transparent encryption / decryption");
3756	MODULE_LICENSE("GPL");
3757