1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Copyright (C) 2012 Red Hat, Inc.
4	*
5	* Author: Mikulas Patocka <mpatocka@redhat.com>
6	*
7	* Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors
8	*
9	* In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set
10	* default prefetch value. Data are read in "prefetch_cluster" chunks from the
11	* hash device. Setting this greatly improves performance when data and hash
12	* are on the same disk on different partitions on devices with poor random
13	* access behavior.
14	*/
15
16	#include "dm-verity.h"
17	#include "dm-verity-fec.h"
18	#include "dm-verity-verify-sig.h"
19	#include "dm-audit.h"
20	#include <linux/module.h>
21	#include <linux/reboot.h>
22	#include <linux/scatterlist.h>
23	#include <linux/string.h>
24	#include <linux/jump_label.h>
25	#include <linux/security.h>
26
27	#define DM_MSG_PREFIX "verity"
28
29	#define DM_VERITY_ENV_LENGTH 42
30	#define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR"
31
32	#define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144
33	#define DM_VERITY_USE_BH_DEFAULT_BYTES 8192
34
35	#define DM_VERITY_MAX_CORRUPTED_ERRS 100
36
37	#define DM_VERITY_OPT_LOGGING "ignore_corruption"
38	#define DM_VERITY_OPT_RESTART "restart_on_corruption"
39	#define DM_VERITY_OPT_PANIC "panic_on_corruption"
40	#define DM_VERITY_OPT_ERROR_RESTART "restart_on_error"
41	#define DM_VERITY_OPT_ERROR_PANIC "panic_on_error"
42	#define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks"
43	#define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once"
44	#define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet"
45
46	#define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \
47	DM_VERITY_ROOT_HASH_VERIFICATION_OPTS)
48
49	static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE;
50
51	module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644);
52
53	static unsigned int dm_verity_use_bh_bytes[4] = {
54	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_NONE
55	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_RT
56	DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_BE
57	0 // IOPRIO_CLASS_IDLE
58	};
59
60	module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644);
61
62	static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled);
63
64	/* Is at least one dm-verity instance using ahash_tfm instead of shash_tfm? */
65	static DEFINE_STATIC_KEY_FALSE(ahash_enabled);
66
67	struct dm_verity_prefetch_work {
68	struct work_struct work;
69	struct dm_verity *v;
70	unsigned short ioprio;
71	sector_t block;
72	unsigned int n_blocks;
73	};
74
75	/*
76	* Auxiliary structure appended to each dm-bufio buffer. If the value
77	* hash_verified is nonzero, hash of the block has been verified.
78	*
79	* The variable hash_verified is set to 0 when allocating the buffer, then
80	* it can be changed to 1 and it is never reset to 0 again.
81	*
82	* There is no lock around this value, a race condition can at worst cause
83	* that multiple processes verify the hash of the same buffer simultaneously
84	* and write 1 to hash_verified simultaneously.
85	* This condition is harmless, so we don't need locking.
86	*/
87	struct buffer_aux {
88	int hash_verified;
89	};
90
91	/*
92	* Initialize struct buffer_aux for a freshly created buffer.
93	*/
94	static void dm_bufio_alloc_callback(struct dm_buffer *buf)
95	{
96	struct buffer_aux *aux = dm_bufio_get_aux_data(b: buf);
97
98	aux->hash_verified = 0;
99	}
100
101	/*
102	* Translate input sector number to the sector number on the target device.
103	*/
104	static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector)
105	{
106	return dm_target_offset(v->ti, bi_sector);
107	}
108
109	/*
110	* Return hash position of a specified block at a specified tree level
111	* (0 is the lowest level).
112	* The lowest "hash_per_block_bits"-bits of the result denote hash position
113	* inside a hash block. The remaining bits denote location of the hash block.
114	*/
115	static sector_t verity_position_at_level(struct dm_verity *v, sector_t block,
116	int level)
117	{
118	return block >> (level * v->hash_per_block_bits);
119	}
120
121	static int verity_ahash_update(struct dm_verity *v, struct ahash_request *req,
122	const u8 *data, size_t len,
123	struct crypto_wait *wait)
124	{
125	struct scatterlist sg;
126
127	if (likely(!is_vmalloc_addr(data))) {
128	sg_init_one(&sg, data, len);
129	ahash_request_set_crypt(req, src: &sg, NULL, nbytes: len);
130	return crypto_wait_req(err: crypto_ahash_update(req), wait);
131	}
132
133	do {
134	int r;
135	size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data));
136
137	flush_kernel_vmap_range(vaddr: (void *)data, size: this_step);
138	sg_init_table(&sg, 1);
139	sg_set_page(sg: &sg, page: vmalloc_to_page(addr: data), len: this_step, offset_in_page(data));
140	ahash_request_set_crypt(req, src: &sg, NULL, nbytes: this_step);
141	r = crypto_wait_req(err: crypto_ahash_update(req), wait);
142	if (unlikely(r))
143	return r;
144	data += this_step;
145	len -= this_step;
146	} while (len);
147
148	return 0;
149	}
150
151	/*
152	* Wrapper for crypto_ahash_init, which handles verity salting.
153	*/
154	static int verity_ahash_init(struct dm_verity *v, struct ahash_request *req,
155	struct crypto_wait *wait, bool may_sleep)
156	{
157	int r;
158
159	ahash_request_set_tfm(req, tfm: v->ahash_tfm);
160	ahash_request_set_callback(req,
161	flags: may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0,
162	compl: crypto_req_done, data: (void *)wait);
163	crypto_init_wait(wait);
164
165	r = crypto_wait_req(err: crypto_ahash_init(req), wait);
166
167	if (unlikely(r < 0)) {
168	if (r != -ENOMEM)
169	DMERR("crypto_ahash_init failed: %d", r);
170	return r;
171	}
172
173	if (likely(v->salt_size && (v->version >= 1)))
174	r = verity_ahash_update(v, req, data: v->salt, len: v->salt_size, wait);
175
176	return r;
177	}
178
179	static int verity_ahash_final(struct dm_verity *v, struct ahash_request *req,
180	u8 *digest, struct crypto_wait *wait)
181	{
182	int r;
183
184	if (unlikely(v->salt_size && (!v->version))) {
185	r = verity_ahash_update(v, req, data: v->salt, len: v->salt_size, wait);
186
187	if (r < 0) {
188	DMERR("%s failed updating salt: %d", __func__, r);
189	goto out;
190	}
191	}
192
193	ahash_request_set_crypt(req, NULL, result: digest, nbytes: 0);
194	r = crypto_wait_req(err: crypto_ahash_final(req), wait);
195	out:
196	return r;
197	}
198
199	int verity_hash(struct dm_verity *v, struct dm_verity_io *io,
200	const u8 *data, size_t len, u8 *digest, bool may_sleep)
201	{
202	int r;
203
204	if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) {
205	struct ahash_request *req = verity_io_hash_req(v, io);
206	struct crypto_wait wait;
207
208	r = verity_ahash_init(v, req, wait: &wait, may_sleep) ?:
209	verity_ahash_update(v, req, data, len, wait: &wait) ?:
210	verity_ahash_final(v, req, digest, wait: &wait);
211	} else {
212	struct shash_desc *desc = verity_io_hash_req(v, io);
213
214	desc->tfm = v->shash_tfm;
215	r = crypto_shash_import(desc, in: v->initial_hashstate) ?:
216	crypto_shash_finup(desc, data, len, out: digest);
217	}
218	if (unlikely(r))
219	DMERR("Error hashing block: %d", r);
220	return r;
221	}
222
223	static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level,
224	sector_t *hash_block, unsigned int *offset)
225	{
226	sector_t position = verity_position_at_level(v, block, level);
227	unsigned int idx;
228
229	*hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits);
230
231	if (!offset)
232	return;
233
234	idx = position & ((1 << v->hash_per_block_bits) - 1);
235	if (!v->version)
236	*offset = idx * v->digest_size;
237	else
238	*offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits);
239	}
240
241	/*
242	* Handle verification errors.
243	*/
244	static int verity_handle_err(struct dm_verity *v, enum verity_block_type type,
245	unsigned long long block)
246	{
247	char verity_env[DM_VERITY_ENV_LENGTH];
248	char *envp[] = { verity_env, NULL };
249	const char *type_str = "";
250	struct mapped_device *md = dm_table_get_md(t: v->ti->table);
251
252	/* Corruption should be visible in device status in all modes */
253	v->hash_failed = true;
254
255	if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS)
256	goto out;
257
258	v->corrupted_errs++;
259
260	switch (type) {
261	case DM_VERITY_BLOCK_TYPE_DATA:
262	type_str = "data";
263	break;
264	case DM_VERITY_BLOCK_TYPE_METADATA:
265	type_str = "metadata";
266	break;
267	default:
268	BUG();
269	}
270
271	DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name,
272	type_str, block);
273
274	if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) {
275	DMERR("%s: reached maximum errors", v->data_dev->name);
276	dm_audit_log_target(DM_MSG_PREFIX, op: "max-corrupted-errors", ti: v->ti, result: 0);
277	}
278
279	snprintf(buf: verity_env, DM_VERITY_ENV_LENGTH, fmt: "%s=%d,%llu",
280	DM_VERITY_ENV_VAR_NAME, type, block);
281
282	kobject_uevent_env(kobj: &disk_to_dev(dm_disk(md))->kobj, action: KOBJ_CHANGE, envp);
283
284	out:
285	if (v->mode == DM_VERITY_MODE_LOGGING)
286	return 0;
287
288	if (v->mode == DM_VERITY_MODE_RESTART)
289	kernel_restart(cmd: "dm-verity device corrupted");
290
291	if (v->mode == DM_VERITY_MODE_PANIC)
292	panic(fmt: "dm-verity device corrupted");
293
294	return 1;
295	}
296
297	/*
298	* Verify hash of a metadata block pertaining to the specified data block
299	* ("block" argument) at a specified level ("level" argument).
300	*
301	* On successful return, verity_io_want_digest(v, io) contains the hash value
302	* for a lower tree level or for the data block (if we're at the lowest level).
303	*
304	* If "skip_unverified" is true, unverified buffer is skipped and 1 is returned.
305	* If "skip_unverified" is false, unverified buffer is hashed and verified
306	* against current value of verity_io_want_digest(v, io).
307	*/
308	static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io,
309	sector_t block, int level, bool skip_unverified,
310	u8 *want_digest)
311	{
312	struct dm_buffer *buf;
313	struct buffer_aux *aux;
314	u8 *data;
315	int r;
316	sector_t hash_block;
317	unsigned int offset;
318	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
319
320	verity_hash_at_level(v, block, level, hash_block: &hash_block, offset: &offset);
321
322	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
323	data = dm_bufio_get(c: v->bufio, block: hash_block, bp: &buf);
324	if (IS_ERR_OR_NULL(ptr: data)) {
325	/*
326	* In tasklet and the hash was not in the bufio cache.
327	* Return early and resume execution from a work-queue
328	* to read the hash from disk.
329	*/
330	return -EAGAIN;
331	}
332	} else {
333	data = dm_bufio_read_with_ioprio(c: v->bufio, block: hash_block,
334	bp: &buf, ioprio: bio->bi_ioprio);
335	}
336
337	if (IS_ERR(ptr: data)) {
338	if (skip_unverified)
339	return 1;
340	r = PTR_ERR(ptr: data);
341	data = dm_bufio_new(c: v->bufio, block: hash_block, bp: &buf);
342	if (IS_ERR(ptr: data))
343	return r;
344	if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA,
345	block: hash_block, dest: data) == 0) {
346	aux = dm_bufio_get_aux_data(b: buf);
347	aux->hash_verified = 1;
348	goto release_ok;
349	} else {
350	dm_bufio_release(b: buf);
351	dm_bufio_forget(c: v->bufio, block: hash_block);
352	return r;
353	}
354	}
355
356	aux = dm_bufio_get_aux_data(b: buf);
357
358	if (!aux->hash_verified) {
359	if (skip_unverified) {
360	r = 1;
361	goto release_ret_r;
362	}
363
364	r = verity_hash(v, io, data, len: 1 << v->hash_dev_block_bits,
365	digest: verity_io_real_digest(v, io), may_sleep: !io->in_bh);
366	if (unlikely(r < 0))
367	goto release_ret_r;
368
369	if (likely(memcmp(verity_io_real_digest(v, io), want_digest,
370	v->digest_size) == 0))
371	aux->hash_verified = 1;
372	else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
373	/*
374	* Error handling code (FEC included) cannot be run in a
375	* tasklet since it may sleep, so fallback to work-queue.
376	*/
377	r = -EAGAIN;
378	goto release_ret_r;
379	} else if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA,
380	block: hash_block, dest: data) == 0)
381	aux->hash_verified = 1;
382	else if (verity_handle_err(v,
383	type: DM_VERITY_BLOCK_TYPE_METADATA,
384	block: hash_block)) {
385	struct bio *bio;
386	io->had_mismatch = true;
387	bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
388	dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-metadata", bio,
389	sector: block, result: 0);
390	r = -EIO;
391	goto release_ret_r;
392	}
393	}
394
395	release_ok:
396	data += offset;
397	memcpy(want_digest, data, v->digest_size);
398	r = 0;
399
400	release_ret_r:
401	dm_bufio_release(b: buf);
402	return r;
403	}
404
405	/*
406	* Find a hash for a given block, write it to digest and verify the integrity
407	* of the hash tree if necessary.
408	*/
409	int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io,
410	sector_t block, u8 *digest, bool *is_zero)
411	{
412	int r = 0, i;
413
414	if (likely(v->levels)) {
415	/*
416	* First, we try to get the requested hash for
417	* the current block. If the hash block itself is
418	* verified, zero is returned. If it isn't, this
419	* function returns 1 and we fall back to whole
420	* chain verification.
421	*/
422	r = verity_verify_level(v, io, block, level: 0, skip_unverified: true, want_digest: digest);
423	if (likely(r <= 0))
424	goto out;
425	}
426
427	memcpy(digest, v->root_digest, v->digest_size);
428
429	for (i = v->levels - 1; i >= 0; i--) {
430	r = verity_verify_level(v, io, block, level: i, skip_unverified: false, want_digest: digest);
431	if (unlikely(r))
432	goto out;
433	}
434	out:
435	if (!r && v->zero_digest)
436	*is_zero = !memcmp(p: v->zero_digest, q: digest, size: v->digest_size);
437	else
438	*is_zero = false;
439
440	return r;
441	}
442
443	static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io,
444	sector_t cur_block, u8 *dest)
445	{
446	struct page *page;
447	void *buffer;
448	int r;
449	struct dm_io_request io_req;
450	struct dm_io_region io_loc;
451
452	page = mempool_alloc(&v->recheck_pool, GFP_NOIO);
453	buffer = page_to_virt(page);
454
455	io_req.bi_opf = REQ_OP_READ;
456	io_req.mem.type = DM_IO_KMEM;
457	io_req.mem.ptr.addr = buffer;
458	io_req.notify.fn = NULL;
459	io_req.client = v->io;
460	io_loc.bdev = v->data_dev->bdev;
461	io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT);
462	io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT);
463	r = dm_io(io_req: &io_req, num_regions: 1, region: &io_loc, NULL, IOPRIO_DEFAULT);
464	if (unlikely(r))
465	goto free_ret;
466
467	r = verity_hash(v, io, data: buffer, len: 1 << v->data_dev_block_bits,
468	digest: verity_io_real_digest(v, io), may_sleep: true);
469	if (unlikely(r))
470	goto free_ret;
471
472	if (memcmp(p: verity_io_real_digest(v, io),
473	q: verity_io_want_digest(v, io), size: v->digest_size)) {
474	r = -EIO;
475	goto free_ret;
476	}
477
478	memcpy(dest, buffer, 1 << v->data_dev_block_bits);
479	r = 0;
480	free_ret:
481	mempool_free(element: page, pool: &v->recheck_pool);
482
483	return r;
484	}
485
486	static int verity_handle_data_hash_mismatch(struct dm_verity *v,
487	struct dm_verity_io *io,
488	struct bio *bio, sector_t blkno,
489	u8 *data)
490	{
491	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
492	/*
493	* Error handling code (FEC included) cannot be run in the
494	* BH workqueue, so fallback to a standard workqueue.
495	*/
496	return -EAGAIN;
497	}
498	if (verity_recheck(v, io, cur_block: blkno, dest: data) == 0) {
499	if (v->validated_blocks)
500	set_bit(nr: blkno, addr: v->validated_blocks);
501	return 0;
502	}
503	#if defined(CONFIG_DM_VERITY_FEC)
504	if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_DATA, block: blkno,
505	dest: data) == 0)
506	return 0;
507	#endif
508	if (bio->bi_status)
509	return -EIO; /* Error correction failed; Just return error */
510
511	if (verity_handle_err(v, type: DM_VERITY_BLOCK_TYPE_DATA, block: blkno)) {
512	io->had_mismatch = true;
513	dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-data", bio, sector: blkno, result: 0);
514	return -EIO;
515	}
516	return 0;
517	}
518
519	/*
520	* Verify one "dm_verity_io" structure.
521	*/
522	static int verity_verify_io(struct dm_verity_io *io)
523	{
524	struct dm_verity *v = io->v;
525	const unsigned int block_size = 1 << v->data_dev_block_bits;
526	struct bvec_iter iter_copy;
527	struct bvec_iter *iter;
528	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
529	unsigned int b;
530
531	if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) {
532	/*
533	* Copy the iterator in case we need to restart
534	* verification in a work-queue.
535	*/
536	iter_copy = io->iter;
537	iter = &iter_copy;
538	} else
539	iter = &io->iter;
540
541	for (b = 0; b < io->n_blocks;
542	b++, bio_advance_iter(bio, iter, bytes: block_size)) {
543	int r;
544	sector_t cur_block = io->block + b;
545	bool is_zero;
546	struct bio_vec bv;
547	void *data;
548
549	if (v->validated_blocks && bio->bi_status == BLK_STS_OK &&
550	likely(test_bit(cur_block, v->validated_blocks)))
551	continue;
552
553	r = verity_hash_for_block(v, io, block: cur_block,
554	digest: verity_io_want_digest(v, io),
555	is_zero: &is_zero);
556	if (unlikely(r < 0))
557	return r;
558
559	bv = bio_iter_iovec(bio, *iter);
560	if (unlikely(bv.bv_len < block_size)) {
561	/*
562	* Data block spans pages. This should not happen,
563	* since dm-verity sets dma_alignment to the data block
564	* size minus 1, and dm-verity also doesn't allow the
565	* data block size to be greater than PAGE_SIZE.
566	*/
567	DMERR_LIMIT("unaligned io (data block spans pages)");
568	return -EIO;
569	}
570
571	data = bvec_kmap_local(bvec: &bv);
572
573	if (is_zero) {
574	/*
575	* If we expect a zero block, don't validate, just
576	* return zeros.
577	*/
578	memset(data, 0, block_size);
579	kunmap_local(data);
580	continue;
581	}
582
583	r = verity_hash(v, io, data, len: block_size,
584	digest: verity_io_real_digest(v, io), may_sleep: !io->in_bh);
585	if (unlikely(r < 0)) {
586	kunmap_local(data);
587	return r;
588	}
589
590	if (likely(memcmp(verity_io_real_digest(v, io),
591	verity_io_want_digest(v, io), v->digest_size) == 0)) {
592	if (v->validated_blocks)
593	set_bit(nr: cur_block, addr: v->validated_blocks);
594	kunmap_local(data);
595	continue;
596	}
597	r = verity_handle_data_hash_mismatch(v, io, bio, blkno: cur_block,
598	data);
599	kunmap_local(data);
600	if (unlikely(r))
601	return r;
602	}
603
604	return 0;
605	}
606
607	/*
608	* Skip verity work in response to I/O error when system is shutting down.
609	*/
610	static inline bool verity_is_system_shutting_down(void)
611	{
612	return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF
613	|| system_state == SYSTEM_RESTART;
614	}
615
616	static void restart_io_error(struct work_struct *w)
617	{
618	kernel_restart(cmd: "dm-verity device has I/O error");
619	}
620
621	/*
622	* End one "io" structure with a given error.
623	*/
624	static void verity_finish_io(struct dm_verity_io *io, blk_status_t status)
625	{
626	struct dm_verity *v = io->v;
627	struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size);
628
629	bio->bi_end_io = io->orig_bi_end_io;
630	bio->bi_status = status;
631
632	if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh)
633	verity_fec_finish_io(io);
634
635	if (unlikely(status != BLK_STS_OK) &&
636	unlikely(!(bio->bi_opf & REQ_RAHEAD)) &&
637	!io->had_mismatch &&
638	!verity_is_system_shutting_down()) {
639	if (v->error_mode == DM_VERITY_MODE_PANIC) {
640	panic(fmt: "dm-verity device has I/O error");
641	}
642	if (v->error_mode == DM_VERITY_MODE_RESTART) {
643	static DECLARE_WORK(restart_work, restart_io_error);
644	queue_work(wq: v->verify_wq, work: &restart_work);
645	/*
646	* We deliberately don't call bio_endio here, because
647	* the machine will be restarted anyway.
648	*/
649	return;
650	}
651	}
652
653	bio_endio(bio);
654	}
655
656	static void verity_work(struct work_struct *w)
657	{
658	struct dm_verity_io *io = container_of(w, struct dm_verity_io, work);
659
660	io->in_bh = false;
661
662	verity_finish_io(io, status: errno_to_blk_status(errno: verity_verify_io(io)));
663	}
664
665	static void verity_bh_work(struct work_struct *w)
666	{
667	struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work);
668	int err;
669
670	io->in_bh = true;
671	err = verity_verify_io(io);
672	if (err == -EAGAIN || err == -ENOMEM) {
673	/* fallback to retrying with work-queue */
674	INIT_WORK(&io->work, verity_work);
675	queue_work(wq: io->v->verify_wq, work: &io->work);
676	return;
677	}
678
679	verity_finish_io(io, status: errno_to_blk_status(errno: err));
680	}
681
682	static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio)
683	{
684	return ioprio <= IOPRIO_CLASS_IDLE &&
685	bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]) &&
686	!need_resched();
687	}
688
689	static void verity_end_io(struct bio *bio)
690	{
691	struct dm_verity_io *io = bio->bi_private;
692	unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio);
693	unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits;
694
695	if (bio->bi_status &&
696	(!verity_fec_is_enabled(v: io->v) ||
697	verity_is_system_shutting_down() ||
698	(bio->bi_opf & REQ_RAHEAD))) {
699	verity_finish_io(io, status: bio->bi_status);
700	return;
701	}
702
703	if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq &&
704	verity_use_bh(bytes, ioprio)) {
705	if (in_hardirq() || irqs_disabled()) {
706	INIT_WORK(&io->bh_work, verity_bh_work);
707	queue_work(wq: system_bh_wq, work: &io->bh_work);
708	} else {
709	verity_bh_work(w: &io->bh_work);
710	}
711	} else {
712	INIT_WORK(&io->work, verity_work);
713	queue_work(wq: io->v->verify_wq, work: &io->work);
714	}
715	}
716
717	/*
718	* Prefetch buffers for the specified io.
719	* The root buffer is not prefetched, it is assumed that it will be cached
720	* all the time.
721	*/
722	static void verity_prefetch_io(struct work_struct *work)
723	{
724	struct dm_verity_prefetch_work *pw =
725	container_of(work, struct dm_verity_prefetch_work, work);
726	struct dm_verity *v = pw->v;
727	int i;
728
729	for (i = v->levels - 2; i >= 0; i--) {
730	sector_t hash_block_start;
731	sector_t hash_block_end;
732
733	verity_hash_at_level(v, block: pw->block, level: i, hash_block: &hash_block_start, NULL);
734	verity_hash_at_level(v, block: pw->block + pw->n_blocks - 1, level: i, hash_block: &hash_block_end, NULL);
735
736	if (!i) {
737	unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster);
738
739	cluster >>= v->data_dev_block_bits;
740	if (unlikely(!cluster))
741	goto no_prefetch_cluster;
742
743	if (unlikely(cluster & (cluster - 1)))
744	cluster = 1 << __fls(word: cluster);
745
746	hash_block_start &= ~(sector_t)(cluster - 1);
747	hash_block_end |= cluster - 1;
748	if (unlikely(hash_block_end >= v->hash_blocks))
749	hash_block_end = v->hash_blocks - 1;
750	}
751	no_prefetch_cluster:
752	dm_bufio_prefetch_with_ioprio(c: v->bufio, block: hash_block_start,
753	n_blocks: hash_block_end - hash_block_start + 1,
754	ioprio: pw->ioprio);
755	}
756
757	kfree(objp: pw);
758	}
759
760	static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io,
761	unsigned short ioprio)
762	{
763	sector_t block = io->block;
764	unsigned int n_blocks = io->n_blocks;
765	struct dm_verity_prefetch_work *pw;
766
767	if (v->validated_blocks) {
768	while (n_blocks && test_bit(block, v->validated_blocks)) {
769	block++;
770	n_blocks--;
771	}
772	while (n_blocks && test_bit(block + n_blocks - 1,
773	v->validated_blocks))
774	n_blocks--;
775	if (!n_blocks)
776	return;
777	}
778
779	pw = kmalloc(sizeof(struct dm_verity_prefetch_work),
780	GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
781
782	if (!pw)
783	return;
784
785	INIT_WORK(&pw->work, verity_prefetch_io);
786	pw->v = v;
787	pw->block = block;
788	pw->n_blocks = n_blocks;
789	pw->ioprio = ioprio;
790	queue_work(wq: v->verify_wq, work: &pw->work);
791	}
792
793	/*
794	* Bio map function. It allocates dm_verity_io structure and bio vector and
795	* fills them. Then it issues prefetches and the I/O.
796	*/
797	static int verity_map(struct dm_target *ti, struct bio *bio)
798	{
799	struct dm_verity *v = ti->private;
800	struct dm_verity_io *io;
801
802	bio_set_dev(bio, bdev: v->data_dev->bdev);
803	bio->bi_iter.bi_sector = verity_map_sector(v, bi_sector: bio->bi_iter.bi_sector);
804
805	if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) &
806	((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) {
807	DMERR_LIMIT("unaligned io");
808	return DM_MAPIO_KILL;
809	}
810
811	if (bio_end_sector(bio) >>
812	(v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) {
813	DMERR_LIMIT("io out of range");
814	return DM_MAPIO_KILL;
815	}
816
817	if (bio_data_dir(bio) == WRITE)
818	return DM_MAPIO_KILL;
819
820	io = dm_per_bio_data(bio, data_size: ti->per_io_data_size);
821	io->v = v;
822	io->orig_bi_end_io = bio->bi_end_io;
823	io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT);
824	io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits;
825	io->had_mismatch = false;
826
827	bio->bi_end_io = verity_end_io;
828	bio->bi_private = io;
829	io->iter = bio->bi_iter;
830
831	verity_fec_init_io(io);
832
833	verity_submit_prefetch(v, io, ioprio: bio->bi_ioprio);
834
835	submit_bio_noacct(bio);
836
837	return DM_MAPIO_SUBMITTED;
838	}
839
840	static void verity_postsuspend(struct dm_target *ti)
841	{
842	struct dm_verity *v = ti->private;
843	flush_workqueue(v->verify_wq);
844	dm_bufio_client_reset(c: v->bufio);
845	}
846
847	/*
848	* Status: V (valid) or C (corruption found)
849	*/
850	static void verity_status(struct dm_target *ti, status_type_t type,
851	unsigned int status_flags, char *result, unsigned int maxlen)
852	{
853	struct dm_verity *v = ti->private;
854	unsigned int args = 0;
855	unsigned int sz = 0;
856	unsigned int x;
857
858	switch (type) {
859	case STATUSTYPE_INFO:
860	DMEMIT("%c", v->hash_failed ? 'C' : 'V');
861	break;
862	case STATUSTYPE_TABLE:
863	DMEMIT("%u %s %s %u %u %llu %llu %s ",
864	v->version,
865	v->data_dev->name,
866	v->hash_dev->name,
867	1 << v->data_dev_block_bits,
868	1 << v->hash_dev_block_bits,
869	(unsigned long long)v->data_blocks,
870	(unsigned long long)v->hash_start,
871	v->alg_name
872	);
873	for (x = 0; x < v->digest_size; x++)
874	DMEMIT("%02x", v->root_digest[x]);
875	DMEMIT(" ");
876	if (!v->salt_size)
877	DMEMIT("-");
878	else
879	for (x = 0; x < v->salt_size; x++)
880	DMEMIT("%02x", v->salt[x]);
881	if (v->mode != DM_VERITY_MODE_EIO)
882	args++;
883	if (v->error_mode != DM_VERITY_MODE_EIO)
884	args++;
885	if (verity_fec_is_enabled(v))
886	args += DM_VERITY_OPTS_FEC;
887	if (v->zero_digest)
888	args++;
889	if (v->validated_blocks)
890	args++;
891	if (v->use_bh_wq)
892	args++;
893	if (v->signature_key_desc)
894	args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS;
895	if (!args)
896	return;
897	DMEMIT(" %u", args);
898	if (v->mode != DM_VERITY_MODE_EIO) {
899	DMEMIT(" ");
900	switch (v->mode) {
901	case DM_VERITY_MODE_LOGGING:
902	DMEMIT(DM_VERITY_OPT_LOGGING);
903	break;
904	case DM_VERITY_MODE_RESTART:
905	DMEMIT(DM_VERITY_OPT_RESTART);
906	break;
907	case DM_VERITY_MODE_PANIC:
908	DMEMIT(DM_VERITY_OPT_PANIC);
909	break;
910	default:
911	BUG();
912	}
913	}
914	if (v->error_mode != DM_VERITY_MODE_EIO) {
915	DMEMIT(" ");
916	switch (v->error_mode) {
917	case DM_VERITY_MODE_RESTART:
918	DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
919	break;
920	case DM_VERITY_MODE_PANIC:
921	DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
922	break;
923	default:
924	BUG();
925	}
926	}
927	if (v->zero_digest)
928	DMEMIT(" " DM_VERITY_OPT_IGN_ZEROES);
929	if (v->validated_blocks)
930	DMEMIT(" " DM_VERITY_OPT_AT_MOST_ONCE);
931	if (v->use_bh_wq)
932	DMEMIT(" " DM_VERITY_OPT_TASKLET_VERIFY);
933	sz = verity_fec_status_table(v, sz, result, maxlen);
934	if (v->signature_key_desc)
935	DMEMIT(" " DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY
936	" %s", v->signature_key_desc);
937	break;
938
939	case STATUSTYPE_IMA:
940	DMEMIT_TARGET_NAME_VERSION(ti->type);
941	DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V');
942	DMEMIT(",verity_version=%u", v->version);
943	DMEMIT(",data_device_name=%s", v->data_dev->name);
944	DMEMIT(",hash_device_name=%s", v->hash_dev->name);
945	DMEMIT(",verity_algorithm=%s", v->alg_name);
946
947	DMEMIT(",root_digest=");
948	for (x = 0; x < v->digest_size; x++)
949	DMEMIT("%02x", v->root_digest[x]);
950
951	DMEMIT(",salt=");
952	if (!v->salt_size)
953	DMEMIT("-");
954	else
955	for (x = 0; x < v->salt_size; x++)
956	DMEMIT("%02x", v->salt[x]);
957
958	DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n');
959	DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n');
960	if (v->signature_key_desc)
961	DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc);
962
963	if (v->mode != DM_VERITY_MODE_EIO) {
964	DMEMIT(",verity_mode=");
965	switch (v->mode) {
966	case DM_VERITY_MODE_LOGGING:
967	DMEMIT(DM_VERITY_OPT_LOGGING);
968	break;
969	case DM_VERITY_MODE_RESTART:
970	DMEMIT(DM_VERITY_OPT_RESTART);
971	break;
972	case DM_VERITY_MODE_PANIC:
973	DMEMIT(DM_VERITY_OPT_PANIC);
974	break;
975	default:
976	DMEMIT("invalid");
977	}
978	}
979	if (v->error_mode != DM_VERITY_MODE_EIO) {
980	DMEMIT(",verity_error_mode=");
981	switch (v->error_mode) {
982	case DM_VERITY_MODE_RESTART:
983	DMEMIT(DM_VERITY_OPT_ERROR_RESTART);
984	break;
985	case DM_VERITY_MODE_PANIC:
986	DMEMIT(DM_VERITY_OPT_ERROR_PANIC);
987	break;
988	default:
989	DMEMIT("invalid");
990	}
991	}
992	DMEMIT(";");
993	break;
994	}
995	}
996
997	static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev,
998	unsigned int cmd, unsigned long arg,
999	bool *forward)
1000	{
1001	struct dm_verity *v = ti->private;
1002
1003	*bdev = v->data_dev->bdev;
1004
1005	if (ti->len != bdev_nr_sectors(bdev: v->data_dev->bdev))
1006	return 1;
1007	return 0;
1008	}
1009
1010	static int verity_iterate_devices(struct dm_target *ti,
1011	iterate_devices_callout_fn fn, void *data)
1012	{
1013	struct dm_verity *v = ti->private;
1014
1015	return fn(ti, v->data_dev, 0, ti->len, data);
1016	}
1017
1018	static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits)
1019	{
1020	struct dm_verity *v = ti->private;
1021
1022	if (limits->logical_block_size < 1 << v->data_dev_block_bits)
1023	limits->logical_block_size = 1 << v->data_dev_block_bits;
1024
1025	if (limits->physical_block_size < 1 << v->data_dev_block_bits)
1026	limits->physical_block_size = 1 << v->data_dev_block_bits;
1027
1028	limits->io_min = limits->logical_block_size;
1029
1030	/*
1031	* Similar to what dm-crypt does, opt dm-verity out of support for
1032	* direct I/O that is aligned to less than the traditional direct I/O
1033	* alignment requirement of logical_block_size. This prevents dm-verity
1034	* data blocks from crossing pages, eliminating various edge cases.
1035	*/
1036	limits->dma_alignment = limits->logical_block_size - 1;
1037	}
1038
1039	#ifdef CONFIG_SECURITY
1040
1041	static int verity_init_sig(struct dm_verity *v, const void *sig,
1042	size_t sig_size)
1043	{
1044	v->sig_size = sig_size;
1045
1046	if (sig) {
1047	v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL);
1048	if (!v->root_digest_sig)
1049	return -ENOMEM;
1050	}
1051
1052	return 0;
1053	}
1054
1055	static void verity_free_sig(struct dm_verity *v)
1056	{
1057	kfree(objp: v->root_digest_sig);
1058	}
1059
1060	#else
1061
1062	static inline int verity_init_sig(struct dm_verity *v, const void *sig,
1063	size_t sig_size)
1064	{
1065	return 0;
1066	}
1067
1068	static inline void verity_free_sig(struct dm_verity *v)
1069	{
1070	}
1071
1072	#endif /* CONFIG_SECURITY */
1073
1074	static void verity_dtr(struct dm_target *ti)
1075	{
1076	struct dm_verity *v = ti->private;
1077
1078	if (v->verify_wq)
1079	destroy_workqueue(wq: v->verify_wq);
1080
1081	mempool_exit(pool: &v->recheck_pool);
1082	if (v->io)
1083	dm_io_client_destroy(client: v->io);
1084
1085	if (v->bufio)
1086	dm_bufio_client_destroy(c: v->bufio);
1087
1088	kvfree(addr: v->validated_blocks);
1089	kfree(objp: v->salt);
1090	kfree(objp: v->initial_hashstate);
1091	kfree(objp: v->root_digest);
1092	kfree(objp: v->zero_digest);
1093	verity_free_sig(v);
1094
1095	if (v->ahash_tfm) {
1096	static_branch_dec(&ahash_enabled);
1097	crypto_free_ahash(tfm: v->ahash_tfm);
1098	} else {
1099	crypto_free_shash(tfm: v->shash_tfm);
1100	}
1101
1102	kfree(objp: v->alg_name);
1103
1104	if (v->hash_dev)
1105	dm_put_device(ti, d: v->hash_dev);
1106
1107	if (v->data_dev)
1108	dm_put_device(ti, d: v->data_dev);
1109
1110	verity_fec_dtr(v);
1111
1112	kfree(objp: v->signature_key_desc);
1113
1114	if (v->use_bh_wq)
1115	static_branch_dec(&use_bh_wq_enabled);
1116
1117	kfree(objp: v);
1118
1119	dm_audit_log_dtr(DM_MSG_PREFIX, ti, result: 1);
1120	}
1121
1122	static int verity_alloc_most_once(struct dm_verity *v)
1123	{
1124	struct dm_target *ti = v->ti;
1125
1126	if (v->validated_blocks)
1127	return 0;
1128
1129	/* the bitset can only handle INT_MAX blocks */
1130	if (v->data_blocks > INT_MAX) {
1131	ti->error = "device too large to use check_at_most_once";
1132	return -E2BIG;
1133	}
1134
1135	v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks),
1136	sizeof(unsigned long),
1137	GFP_KERNEL);
1138	if (!v->validated_blocks) {
1139	ti->error = "failed to allocate bitset for check_at_most_once";
1140	return -ENOMEM;
1141	}
1142
1143	return 0;
1144	}
1145
1146	static int verity_alloc_zero_digest(struct dm_verity *v)
1147	{
1148	int r = -ENOMEM;
1149	struct dm_verity_io *io;
1150	u8 *zero_data;
1151
1152	if (v->zero_digest)
1153	return 0;
1154
1155	v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL);
1156
1157	if (!v->zero_digest)
1158	return r;
1159
1160	io = kmalloc(sizeof(*io) + v->hash_reqsize, GFP_KERNEL);
1161
1162	if (!io)
1163	return r; /* verity_dtr will free zero_digest */
1164
1165	zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL);
1166
1167	if (!zero_data)
1168	goto out;
1169
1170	r = verity_hash(v, io, data: zero_data, len: 1 << v->data_dev_block_bits,
1171	digest: v->zero_digest, may_sleep: true);
1172
1173	out:
1174	kfree(objp: io);
1175	kfree(objp: zero_data);
1176
1177	return r;
1178	}
1179
1180	static inline bool verity_is_verity_mode(const char *arg_name)
1181	{
1182	return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING) ||
1183	!strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART) ||
1184	!strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC));
1185	}
1186
1187	static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name)
1188	{
1189	if (v->mode)
1190	return -EINVAL;
1191
1192	if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING))
1193	v->mode = DM_VERITY_MODE_LOGGING;
1194	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART))
1195	v->mode = DM_VERITY_MODE_RESTART;
1196	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC))
1197	v->mode = DM_VERITY_MODE_PANIC;
1198
1199	return 0;
1200	}
1201
1202	static inline bool verity_is_verity_error_mode(const char *arg_name)
1203	{
1204	return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART) ||
1205	!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC));
1206	}
1207
1208	static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name)
1209	{
1210	if (v->error_mode)
1211	return -EINVAL;
1212
1213	if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART))
1214	v->error_mode = DM_VERITY_MODE_RESTART;
1215	else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC))
1216	v->error_mode = DM_VERITY_MODE_PANIC;
1217
1218	return 0;
1219	}
1220
1221	static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
1222	struct dm_verity_sig_opts *verify_args,
1223	bool only_modifier_opts)
1224	{
1225	int r = 0;
1226	unsigned int argc;
1227	struct dm_target *ti = v->ti;
1228	const char *arg_name;
1229
1230	static const struct dm_arg _args[] = {
1231	{0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"},
1232	};
1233
1234	r = dm_read_arg_group(arg: _args, arg_set: as, num_args: &argc, error: &ti->error);
1235	if (r)
1236	return -EINVAL;
1237
1238	if (!argc)
1239	return 0;
1240
1241	do {
1242	arg_name = dm_shift_arg(as);
1243	argc--;
1244
1245	if (verity_is_verity_mode(arg_name)) {
1246	if (only_modifier_opts)
1247	continue;
1248	r = verity_parse_verity_mode(v, arg_name);
1249	if (r) {
1250	ti->error = "Conflicting error handling parameters";
1251	return r;
1252	}
1253	continue;
1254
1255	} else if (verity_is_verity_error_mode(arg_name)) {
1256	if (only_modifier_opts)
1257	continue;
1258	r = verity_parse_verity_error_mode(v, arg_name);
1259	if (r) {
1260	ti->error = "Conflicting error handling parameters";
1261	return r;
1262	}
1263	continue;
1264
1265	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_IGN_ZEROES)) {
1266	if (only_modifier_opts)
1267	continue;
1268	r = verity_alloc_zero_digest(v);
1269	if (r) {
1270	ti->error = "Cannot allocate zero digest";
1271	return r;
1272	}
1273	continue;
1274
1275	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) {
1276	if (only_modifier_opts)
1277	continue;
1278	r = verity_alloc_most_once(v);
1279	if (r)
1280	return r;
1281	continue;
1282
1283	} else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) {
1284	v->use_bh_wq = true;
1285	static_branch_inc(&use_bh_wq_enabled);
1286	continue;
1287
1288	} else if (verity_is_fec_opt_arg(arg_name)) {
1289	if (only_modifier_opts)
1290	continue;
1291	r = verity_fec_parse_opt_args(as, v, argc: &argc, arg_name);
1292	if (r)
1293	return r;
1294	continue;
1295
1296	} else if (verity_verify_is_sig_opt_arg(arg_name)) {
1297	if (only_modifier_opts)
1298	continue;
1299	r = verity_verify_sig_parse_opt_args(as, v,
1300	sig_opts: verify_args,
1301	argc: &argc, arg_name);
1302	if (r)
1303	return r;
1304	continue;
1305
1306	} else if (only_modifier_opts) {
1307	/*
1308	* Ignore unrecognized opt, could easily be an extra
1309	* argument to an option whose parsing was skipped.
1310	* Normal parsing (@only_modifier_opts=false) will
1311	* properly parse all options (and their extra args).
1312	*/
1313	continue;
1314	}
1315
1316	DMERR("Unrecognized verity feature request: %s", arg_name);
1317	ti->error = "Unrecognized verity feature request";
1318	return -EINVAL;
1319	} while (argc && !r);
1320
1321	return r;
1322	}
1323
1324	static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name)
1325	{
1326	struct dm_target *ti = v->ti;
1327	struct crypto_ahash *ahash;
1328	struct crypto_shash *shash = NULL;
1329	const char *driver_name;
1330
1331	v->alg_name = kstrdup(s: alg_name, GFP_KERNEL);
1332	if (!v->alg_name) {
1333	ti->error = "Cannot allocate algorithm name";
1334	return -ENOMEM;
1335	}
1336
1337	/*
1338	* Allocate the hash transformation object that this dm-verity instance
1339	* will use. The vast majority of dm-verity users use CPU-based
1340	* hashing, so when possible use the shash API to minimize the crypto
1341	* API overhead. If the ahash API resolves to a different driver
1342	* (likely an off-CPU hardware offload), use ahash instead. Also use
1343	* ahash if the obsolete dm-verity format with the appended salt is
1344	* being used, so that quirk only needs to be handled in one place.
1345	*/
1346	ahash = crypto_alloc_ahash(alg_name, type: 0,
1347	mask: v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0);
1348	if (IS_ERR(ptr: ahash)) {
1349	ti->error = "Cannot initialize hash function";
1350	return PTR_ERR(ptr: ahash);
1351	}
1352	driver_name = crypto_ahash_driver_name(tfm: ahash);
1353	if (v->version >= 1 /* salt prepended, not appended? */) {
1354	shash = crypto_alloc_shash(alg_name, type: 0, mask: 0);
1355	if (!IS_ERR(ptr: shash) &&
1356	strcmp(crypto_shash_driver_name(tfm: shash), driver_name) != 0) {
1357	/*
1358	* ahash gave a different driver than shash, so probably
1359	* this is a case of real hardware offload. Use ahash.
1360	*/
1361	crypto_free_shash(tfm: shash);
1362	shash = NULL;
1363	}
1364	}
1365	if (!IS_ERR_OR_NULL(ptr: shash)) {
1366	crypto_free_ahash(tfm: ahash);
1367	ahash = NULL;
1368	v->shash_tfm = shash;
1369	v->digest_size = crypto_shash_digestsize(tfm: shash);
1370	v->hash_reqsize = sizeof(struct shash_desc) +
1371	crypto_shash_descsize(tfm: shash);
1372	DMINFO("%s using shash \"%s\"", alg_name, driver_name);
1373	} else {
1374	v->ahash_tfm = ahash;
1375	static_branch_inc(&ahash_enabled);
1376	v->digest_size = crypto_ahash_digestsize(tfm: ahash);
1377	v->hash_reqsize = sizeof(struct ahash_request) +
1378	crypto_ahash_reqsize(tfm: ahash);
1379	DMINFO("%s using ahash \"%s\"", alg_name, driver_name);
1380	}
1381	if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) {
1382	ti->error = "Digest size too big";
1383	return -EINVAL;
1384	}
1385	return 0;
1386	}
1387
1388	static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg)
1389	{
1390	struct dm_target *ti = v->ti;
1391
1392	if (strcmp(arg, "-") != 0) {
1393	v->salt_size = strlen(arg) / 2;
1394	v->salt = kmalloc(v->salt_size, GFP_KERNEL);
1395	if (!v->salt) {
1396	ti->error = "Cannot allocate salt";
1397	return -ENOMEM;
1398	}
1399	if (strlen(arg) != v->salt_size * 2 ||
1400	hex2bin(dst: v->salt, src: arg, count: v->salt_size)) {
1401	ti->error = "Invalid salt";
1402	return -EINVAL;
1403	}
1404	}
1405	if (v->shash_tfm) {
1406	SHASH_DESC_ON_STACK(desc, v->shash_tfm);
1407	int r;
1408
1409	/*
1410	* Compute the pre-salted hash state that can be passed to
1411	* crypto_shash_import() for each block later.
1412	*/
1413	v->initial_hashstate = kmalloc(
1414	crypto_shash_statesize(v->shash_tfm), GFP_KERNEL);
1415	if (!v->initial_hashstate) {
1416	ti->error = "Cannot allocate initial hash state";
1417	return -ENOMEM;
1418	}
1419	desc->tfm = v->shash_tfm;
1420	r = crypto_shash_init(desc) ?:
1421	crypto_shash_update(desc, data: v->salt, len: v->salt_size) ?:
1422	crypto_shash_export(desc, out: v->initial_hashstate);
1423	if (r) {
1424	ti->error = "Cannot set up initial hash state";
1425	return r;
1426	}
1427	}
1428	return 0;
1429	}
1430
1431	/*
1432	* Target parameters:
1433	* <version> The current format is version 1.
1434	* Vsn 0 is compatible with original Chromium OS releases.
1435	* <data device>
1436	* <hash device>
1437	* <data block size>
1438	* <hash block size>
1439	* <the number of data blocks>
1440	* <hash start block>
1441	* <algorithm>
1442	* <digest>
1443	* <salt> Hex string or "-" if no salt.
1444	*/
1445	static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1446	{
1447	struct dm_verity *v;
1448	struct dm_verity_sig_opts verify_args = {0};
1449	struct dm_arg_set as;
1450	unsigned int num;
1451	unsigned long long num_ll;
1452	int r;
1453	int i;
1454	sector_t hash_position;
1455	char dummy;
1456	char *root_hash_digest_to_validate;
1457
1458	v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL);
1459	if (!v) {
1460	ti->error = "Cannot allocate verity structure";
1461	return -ENOMEM;
1462	}
1463	ti->private = v;
1464	v->ti = ti;
1465
1466	r = verity_fec_ctr_alloc(v);
1467	if (r)
1468	goto bad;
1469
1470	if ((dm_table_get_mode(t: ti->table) & ~BLK_OPEN_READ)) {
1471	ti->error = "Device must be readonly";
1472	r = -EINVAL;
1473	goto bad;
1474	}
1475
1476	if (argc < 10) {
1477	ti->error = "Not enough arguments";
1478	r = -EINVAL;
1479	goto bad;
1480	}
1481
1482	/* Parse optional parameters that modify primary args */
1483	if (argc > 10) {
1484	as.argc = argc - 10;
1485	as.argv = argv + 10;
1486	r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: true);
1487	if (r < 0)
1488	goto bad;
1489	}
1490
1491	if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 ||
1492	num > 1) {
1493	ti->error = "Invalid version";
1494	r = -EINVAL;
1495	goto bad;
1496	}
1497	v->version = num;
1498
1499	r = dm_get_device(ti, path: argv[1], BLK_OPEN_READ, result: &v->data_dev);
1500	if (r) {
1501	ti->error = "Data device lookup failed";
1502	goto bad;
1503	}
1504
1505	r = dm_get_device(ti, path: argv[2], BLK_OPEN_READ, result: &v->hash_dev);
1506	if (r) {
1507	ti->error = "Hash device lookup failed";
1508	goto bad;
1509	}
1510
1511	if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 ||
1512	!num || (num & (num - 1)) ||
1513	num < bdev_logical_block_size(bdev: v->data_dev->bdev) ||
1514	num > PAGE_SIZE) {
1515	ti->error = "Invalid data device block size";
1516	r = -EINVAL;
1517	goto bad;
1518	}
1519	v->data_dev_block_bits = __ffs(num);
1520
1521	if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 ||
1522	!num || (num & (num - 1)) ||
1523	num < bdev_logical_block_size(bdev: v->hash_dev->bdev) ||
1524	num > INT_MAX) {
1525	ti->error = "Invalid hash device block size";
1526	r = -EINVAL;
1527	goto bad;
1528	}
1529	v->hash_dev_block_bits = __ffs(num);
1530
1531	if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 ||
1532	(sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
1533	>> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1534	ti->error = "Invalid data blocks";
1535	r = -EINVAL;
1536	goto bad;
1537	}
1538	v->data_blocks = num_ll;
1539
1540	if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) {
1541	ti->error = "Data device is too small";
1542	r = -EINVAL;
1543	goto bad;
1544	}
1545
1546	if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 ||
1547	(sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT))
1548	>> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) {
1549	ti->error = "Invalid hash start";
1550	r = -EINVAL;
1551	goto bad;
1552	}
1553	v->hash_start = num_ll;
1554
1555	r = verity_setup_hash_alg(v, alg_name: argv[7]);
1556	if (r)
1557	goto bad;
1558
1559	v->root_digest = kmalloc(v->digest_size, GFP_KERNEL);
1560	if (!v->root_digest) {
1561	ti->error = "Cannot allocate root digest";
1562	r = -ENOMEM;
1563	goto bad;
1564	}
1565	if (strlen(argv[8]) != v->digest_size * 2 ||
1566	hex2bin(dst: v->root_digest, src: argv[8], count: v->digest_size)) {
1567	ti->error = "Invalid root digest";
1568	r = -EINVAL;
1569	goto bad;
1570	}
1571	root_hash_digest_to_validate = argv[8];
1572
1573	r = verity_setup_salt_and_hashstate(v, arg: argv[9]);
1574	if (r)
1575	goto bad;
1576
1577	argv += 10;
1578	argc -= 10;
1579
1580	/* Optional parameters */
1581	if (argc) {
1582	as.argc = argc;
1583	as.argv = argv;
1584	r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: false);
1585	if (r < 0)
1586	goto bad;
1587	}
1588
1589	/* Root hash signature is an optional parameter */
1590	r = verity_verify_root_hash(data: root_hash_digest_to_validate,
1591	strlen(root_hash_digest_to_validate),
1592	sig_data: verify_args.sig,
1593	sig_len: verify_args.sig_size);
1594	if (r < 0) {
1595	ti->error = "Root hash verification failed";
1596	goto bad;
1597	}
1598
1599	r = verity_init_sig(v, sig: verify_args.sig, sig_size: verify_args.sig_size);
1600	if (r < 0) {
1601	ti->error = "Cannot allocate root digest signature";
1602	goto bad;
1603	}
1604
1605	v->hash_per_block_bits =
1606	__fls(word: (1 << v->hash_dev_block_bits) / v->digest_size);
1607
1608	v->levels = 0;
1609	if (v->data_blocks)
1610	while (v->hash_per_block_bits * v->levels < 64 &&
1611	(unsigned long long)(v->data_blocks - 1) >>
1612	(v->hash_per_block_bits * v->levels))
1613	v->levels++;
1614
1615	if (v->levels > DM_VERITY_MAX_LEVELS) {
1616	ti->error = "Too many tree levels";
1617	r = -E2BIG;
1618	goto bad;
1619	}
1620
1621	hash_position = v->hash_start;
1622	for (i = v->levels - 1; i >= 0; i--) {
1623	sector_t s;
1624
1625	v->hash_level_block[i] = hash_position;
1626	s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1)
1627	>> ((i + 1) * v->hash_per_block_bits);
1628	if (hash_position + s < hash_position) {
1629	ti->error = "Hash device offset overflow";
1630	r = -E2BIG;
1631	goto bad;
1632	}
1633	hash_position += s;
1634	}
1635	v->hash_blocks = hash_position;
1636
1637	r = mempool_init_page_pool(&v->recheck_pool, 1, 0);
1638	if (unlikely(r)) {
1639	ti->error = "Cannot allocate mempool";
1640	goto bad;
1641	}
1642
1643	v->io = dm_io_client_create();
1644	if (IS_ERR(ptr: v->io)) {
1645	r = PTR_ERR(ptr: v->io);
1646	v->io = NULL;
1647	ti->error = "Cannot allocate dm io";
1648	goto bad;
1649	}
1650
1651	v->bufio = dm_bufio_client_create(bdev: v->hash_dev->bdev,
1652	block_size: 1 << v->hash_dev_block_bits, reserved_buffers: 1, aux_size: sizeof(struct buffer_aux),
1653	alloc_callback: dm_bufio_alloc_callback, NULL,
1654	flags: v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0);
1655	if (IS_ERR(ptr: v->bufio)) {
1656	ti->error = "Cannot initialize dm-bufio";
1657	r = PTR_ERR(ptr: v->bufio);
1658	v->bufio = NULL;
1659	goto bad;
1660	}
1661
1662	if (dm_bufio_get_device_size(c: v->bufio) < v->hash_blocks) {
1663	ti->error = "Hash device is too small";
1664	r = -E2BIG;
1665	goto bad;
1666	}
1667
1668	/*
1669	* Using WQ_HIGHPRI improves throughput and completion latency by
1670	* reducing wait times when reading from a dm-verity device.
1671	*
1672	* Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI
1673	* allows verify_wq to preempt softirq since verification in BH workqueue
1674	* will fall-back to using it for error handling (or if the bufio cache
1675	* doesn't have required hashes).
1676	*/
1677	v->verify_wq = alloc_workqueue(fmt: "kverityd", flags: WQ_MEM_RECLAIM | WQ_HIGHPRI, max_active: 0);
1678	if (!v->verify_wq) {
1679	ti->error = "Cannot allocate workqueue";
1680	r = -ENOMEM;
1681	goto bad;
1682	}
1683
1684	ti->per_io_data_size = sizeof(struct dm_verity_io) + v->hash_reqsize;
1685
1686	r = verity_fec_ctr(v);
1687	if (r)
1688	goto bad;
1689
1690	ti->per_io_data_size = roundup(ti->per_io_data_size,
1691	__alignof__(struct dm_verity_io));
1692
1693	verity_verify_sig_opts_cleanup(sig_opts: &verify_args);
1694
1695	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 1);
1696
1697	return 0;
1698
1699	bad:
1700
1701	verity_verify_sig_opts_cleanup(sig_opts: &verify_args);
1702	dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 0);
1703	verity_dtr(ti);
1704
1705	return r;
1706	}
1707
1708	/*
1709	* Get the verity mode (error behavior) of a verity target.
1710	*
1711	* Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity
1712	* target.
1713	*/
1714	int dm_verity_get_mode(struct dm_target *ti)
1715	{
1716	struct dm_verity *v = ti->private;
1717
1718	if (!dm_is_verity_target(ti))
1719	return -EINVAL;
1720
1721	return v->mode;
1722	}
1723
1724	/*
1725	* Get the root digest of a verity target.
1726	*
1727	* Returns a copy of the root digest, the caller is responsible for
1728	* freeing the memory of the digest.
1729	*/
1730	int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size)
1731	{
1732	struct dm_verity *v = ti->private;
1733
1734	if (!dm_is_verity_target(ti))
1735	return -EINVAL;
1736
1737	*root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL);
1738	if (*root_digest == NULL)
1739	return -ENOMEM;
1740
1741	*digest_size = v->digest_size;
1742
1743	return 0;
1744	}
1745
1746	#ifdef CONFIG_SECURITY
1747
1748	#ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG
1749
1750	static int verity_security_set_signature(struct block_device *bdev,
1751	struct dm_verity *v)
1752	{
1753	/*
1754	* if the dm-verity target is unsigned, v->root_digest_sig will
1755	* be NULL, and the hook call is still required to let LSMs mark
1756	* the device as unsigned. This information is crucial for LSMs to
1757	* block operations such as execution on unsigned files
1758	*/
1759	return security_bdev_setintegrity(bdev,
1760	type: LSM_INT_DMVERITY_SIG_VALID,
1761	value: v->root_digest_sig,
1762	size: v->sig_size);
1763	}
1764
1765	#else
1766
1767	static inline int verity_security_set_signature(struct block_device *bdev,
1768	struct dm_verity *v)
1769	{
1770	return 0;
1771	}
1772
1773	#endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */
1774
1775	/*
1776	* Expose verity target's root hash and signature data to LSMs before resume.
1777	*
1778	* Returns 0 on success, or -ENOMEM if the system is out of memory.
1779	*/
1780	static int verity_preresume(struct dm_target *ti)
1781	{
1782	struct block_device *bdev;
1783	struct dm_verity_digest root_digest;
1784	struct dm_verity *v;
1785	int r;
1786
1787	v = ti->private;
1788	bdev = dm_disk(md: dm_table_get_md(t: ti->table))->part0;
1789	root_digest.digest = v->root_digest;
1790	root_digest.digest_len = v->digest_size;
1791	if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm)
1792	root_digest.alg = crypto_ahash_alg_name(tfm: v->ahash_tfm);
1793	else
1794	root_digest.alg = crypto_shash_alg_name(tfm: v->shash_tfm);
1795
1796	r = security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, value: &root_digest,
1797	size: sizeof(root_digest));
1798	if (r)
1799	return r;
1800
1801	r = verity_security_set_signature(bdev, v);
1802	if (r)
1803	goto bad;
1804
1805	return 0;
1806
1807	bad:
1808
1809	security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, NULL, size: 0);
1810
1811	return r;
1812	}
1813
1814	#endif /* CONFIG_SECURITY */
1815
1816	static struct target_type verity_target = {
1817	.name = "verity",
1818	/* Note: the LSMs depend on the singleton and immutable features */
1819	.features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1820	.version = {1, 11, 0},
1821	.module = THIS_MODULE,
1822	.ctr = verity_ctr,
1823	.dtr = verity_dtr,
1824	.map = verity_map,
1825	.postsuspend = verity_postsuspend,
1826	.status = verity_status,
1827	.prepare_ioctl = verity_prepare_ioctl,
1828	.iterate_devices = verity_iterate_devices,
1829	.io_hints = verity_io_hints,
1830	#ifdef CONFIG_SECURITY
1831	.preresume = verity_preresume,
1832	#endif /* CONFIG_SECURITY */
1833	};
1834	module_dm(verity);
1835
1836	/*
1837	* Check whether a DM target is a verity target.
1838	*/
1839	bool dm_is_verity_target(struct dm_target *ti)
1840	{
1841	return ti->type == &verity_target;
1842	}
1843
1844	MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>");
1845	MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>");
1846	MODULE_AUTHOR("Will Drewry <wad@chromium.org>");
1847	MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
1848	MODULE_LICENSE("GPL");
1849