1 | // SPDX-License-Identifier: GPL-2.0-only |
---|---|
2 | /* |
3 | * Copyright (C) 2012 Red Hat, Inc. |
4 | * |
5 | * Author: Mikulas Patocka <mpatocka@redhat.com> |
6 | * |
7 | * Based on Chromium dm-verity driver (C) 2011 The Chromium OS Authors |
8 | * |
9 | * In the file "/sys/module/dm_verity/parameters/prefetch_cluster" you can set |
10 | * default prefetch value. Data are read in "prefetch_cluster" chunks from the |
11 | * hash device. Setting this greatly improves performance when data and hash |
12 | * are on the same disk on different partitions on devices with poor random |
13 | * access behavior. |
14 | */ |
15 | |
16 | #include "dm-verity.h" |
17 | #include "dm-verity-fec.h" |
18 | #include "dm-verity-verify-sig.h" |
19 | #include "dm-audit.h" |
20 | #include <linux/module.h> |
21 | #include <linux/reboot.h> |
22 | #include <linux/scatterlist.h> |
23 | #include <linux/string.h> |
24 | #include <linux/jump_label.h> |
25 | #include <linux/security.h> |
26 | |
27 | #define DM_MSG_PREFIX "verity" |
28 | |
29 | #define DM_VERITY_ENV_LENGTH 42 |
30 | #define DM_VERITY_ENV_VAR_NAME "DM_VERITY_ERR_BLOCK_NR" |
31 | |
32 | #define DM_VERITY_DEFAULT_PREFETCH_SIZE 262144 |
33 | #define DM_VERITY_USE_BH_DEFAULT_BYTES 8192 |
34 | |
35 | #define DM_VERITY_MAX_CORRUPTED_ERRS 100 |
36 | |
37 | #define DM_VERITY_OPT_LOGGING "ignore_corruption" |
38 | #define DM_VERITY_OPT_RESTART "restart_on_corruption" |
39 | #define DM_VERITY_OPT_PANIC "panic_on_corruption" |
40 | #define DM_VERITY_OPT_ERROR_RESTART "restart_on_error" |
41 | #define DM_VERITY_OPT_ERROR_PANIC "panic_on_error" |
42 | #define DM_VERITY_OPT_IGN_ZEROES "ignore_zero_blocks" |
43 | #define DM_VERITY_OPT_AT_MOST_ONCE "check_at_most_once" |
44 | #define DM_VERITY_OPT_TASKLET_VERIFY "try_verify_in_tasklet" |
45 | |
46 | #define DM_VERITY_OPTS_MAX (5 + DM_VERITY_OPTS_FEC + \ |
47 | DM_VERITY_ROOT_HASH_VERIFICATION_OPTS) |
48 | |
49 | static unsigned int dm_verity_prefetch_cluster = DM_VERITY_DEFAULT_PREFETCH_SIZE; |
50 | |
51 | module_param_named(prefetch_cluster, dm_verity_prefetch_cluster, uint, 0644); |
52 | |
53 | static unsigned int dm_verity_use_bh_bytes[4] = { |
54 | DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_NONE |
55 | DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_RT |
56 | DM_VERITY_USE_BH_DEFAULT_BYTES, // IOPRIO_CLASS_BE |
57 | 0 // IOPRIO_CLASS_IDLE |
58 | }; |
59 | |
60 | module_param_array_named(use_bh_bytes, dm_verity_use_bh_bytes, uint, NULL, 0644); |
61 | |
62 | static DEFINE_STATIC_KEY_FALSE(use_bh_wq_enabled); |
63 | |
64 | /* Is at least one dm-verity instance using ahash_tfm instead of shash_tfm? */ |
65 | static DEFINE_STATIC_KEY_FALSE(ahash_enabled); |
66 | |
67 | struct dm_verity_prefetch_work { |
68 | struct work_struct work; |
69 | struct dm_verity *v; |
70 | unsigned short ioprio; |
71 | sector_t block; |
72 | unsigned int n_blocks; |
73 | }; |
74 | |
75 | /* |
76 | * Auxiliary structure appended to each dm-bufio buffer. If the value |
77 | * hash_verified is nonzero, hash of the block has been verified. |
78 | * |
79 | * The variable hash_verified is set to 0 when allocating the buffer, then |
80 | * it can be changed to 1 and it is never reset to 0 again. |
81 | * |
82 | * There is no lock around this value, a race condition can at worst cause |
83 | * that multiple processes verify the hash of the same buffer simultaneously |
84 | * and write 1 to hash_verified simultaneously. |
85 | * This condition is harmless, so we don't need locking. |
86 | */ |
87 | struct buffer_aux { |
88 | int hash_verified; |
89 | }; |
90 | |
91 | /* |
92 | * Initialize struct buffer_aux for a freshly created buffer. |
93 | */ |
94 | static void dm_bufio_alloc_callback(struct dm_buffer *buf) |
95 | { |
96 | struct buffer_aux *aux = dm_bufio_get_aux_data(b: buf); |
97 | |
98 | aux->hash_verified = 0; |
99 | } |
100 | |
101 | /* |
102 | * Translate input sector number to the sector number on the target device. |
103 | */ |
104 | static sector_t verity_map_sector(struct dm_verity *v, sector_t bi_sector) |
105 | { |
106 | return dm_target_offset(v->ti, bi_sector); |
107 | } |
108 | |
109 | /* |
110 | * Return hash position of a specified block at a specified tree level |
111 | * (0 is the lowest level). |
112 | * The lowest "hash_per_block_bits"-bits of the result denote hash position |
113 | * inside a hash block. The remaining bits denote location of the hash block. |
114 | */ |
115 | static sector_t verity_position_at_level(struct dm_verity *v, sector_t block, |
116 | int level) |
117 | { |
118 | return block >> (level * v->hash_per_block_bits); |
119 | } |
120 | |
121 | static int verity_ahash_update(struct dm_verity *v, struct ahash_request *req, |
122 | const u8 *data, size_t len, |
123 | struct crypto_wait *wait) |
124 | { |
125 | struct scatterlist sg; |
126 | |
127 | if (likely(!is_vmalloc_addr(data))) { |
128 | sg_init_one(&sg, data, len); |
129 | ahash_request_set_crypt(req, src: &sg, NULL, nbytes: len); |
130 | return crypto_wait_req(err: crypto_ahash_update(req), wait); |
131 | } |
132 | |
133 | do { |
134 | int r; |
135 | size_t this_step = min_t(size_t, len, PAGE_SIZE - offset_in_page(data)); |
136 | |
137 | flush_kernel_vmap_range(vaddr: (void *)data, size: this_step); |
138 | sg_init_table(&sg, 1); |
139 | sg_set_page(sg: &sg, page: vmalloc_to_page(addr: data), len: this_step, offset_in_page(data)); |
140 | ahash_request_set_crypt(req, src: &sg, NULL, nbytes: this_step); |
141 | r = crypto_wait_req(err: crypto_ahash_update(req), wait); |
142 | if (unlikely(r)) |
143 | return r; |
144 | data += this_step; |
145 | len -= this_step; |
146 | } while (len); |
147 | |
148 | return 0; |
149 | } |
150 | |
151 | /* |
152 | * Wrapper for crypto_ahash_init, which handles verity salting. |
153 | */ |
154 | static int verity_ahash_init(struct dm_verity *v, struct ahash_request *req, |
155 | struct crypto_wait *wait, bool may_sleep) |
156 | { |
157 | int r; |
158 | |
159 | ahash_request_set_tfm(req, tfm: v->ahash_tfm); |
160 | ahash_request_set_callback(req, |
161 | flags: may_sleep ? CRYPTO_TFM_REQ_MAY_SLEEP | CRYPTO_TFM_REQ_MAY_BACKLOG : 0, |
162 | compl: crypto_req_done, data: (void *)wait); |
163 | crypto_init_wait(wait); |
164 | |
165 | r = crypto_wait_req(err: crypto_ahash_init(req), wait); |
166 | |
167 | if (unlikely(r < 0)) { |
168 | if (r != -ENOMEM) |
169 | DMERR("crypto_ahash_init failed: %d", r); |
170 | return r; |
171 | } |
172 | |
173 | if (likely(v->salt_size && (v->version >= 1))) |
174 | r = verity_ahash_update(v, req, data: v->salt, len: v->salt_size, wait); |
175 | |
176 | return r; |
177 | } |
178 | |
179 | static int verity_ahash_final(struct dm_verity *v, struct ahash_request *req, |
180 | u8 *digest, struct crypto_wait *wait) |
181 | { |
182 | int r; |
183 | |
184 | if (unlikely(v->salt_size && (!v->version))) { |
185 | r = verity_ahash_update(v, req, data: v->salt, len: v->salt_size, wait); |
186 | |
187 | if (r < 0) { |
188 | DMERR("%s failed updating salt: %d", __func__, r); |
189 | goto out; |
190 | } |
191 | } |
192 | |
193 | ahash_request_set_crypt(req, NULL, result: digest, nbytes: 0); |
194 | r = crypto_wait_req(err: crypto_ahash_final(req), wait); |
195 | out: |
196 | return r; |
197 | } |
198 | |
199 | int verity_hash(struct dm_verity *v, struct dm_verity_io *io, |
200 | const u8 *data, size_t len, u8 *digest, bool may_sleep) |
201 | { |
202 | int r; |
203 | |
204 | if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) { |
205 | struct ahash_request *req = verity_io_hash_req(v, io); |
206 | struct crypto_wait wait; |
207 | |
208 | r = verity_ahash_init(v, req, wait: &wait, may_sleep) ?: |
209 | verity_ahash_update(v, req, data, len, wait: &wait) ?: |
210 | verity_ahash_final(v, req, digest, wait: &wait); |
211 | } else { |
212 | struct shash_desc *desc = verity_io_hash_req(v, io); |
213 | |
214 | desc->tfm = v->shash_tfm; |
215 | r = crypto_shash_import(desc, in: v->initial_hashstate) ?: |
216 | crypto_shash_finup(desc, data, len, out: digest); |
217 | } |
218 | if (unlikely(r)) |
219 | DMERR("Error hashing block: %d", r); |
220 | return r; |
221 | } |
222 | |
223 | static void verity_hash_at_level(struct dm_verity *v, sector_t block, int level, |
224 | sector_t *hash_block, unsigned int *offset) |
225 | { |
226 | sector_t position = verity_position_at_level(v, block, level); |
227 | unsigned int idx; |
228 | |
229 | *hash_block = v->hash_level_block[level] + (position >> v->hash_per_block_bits); |
230 | |
231 | if (!offset) |
232 | return; |
233 | |
234 | idx = position & ((1 << v->hash_per_block_bits) - 1); |
235 | if (!v->version) |
236 | *offset = idx * v->digest_size; |
237 | else |
238 | *offset = idx << (v->hash_dev_block_bits - v->hash_per_block_bits); |
239 | } |
240 | |
241 | /* |
242 | * Handle verification errors. |
243 | */ |
244 | static int verity_handle_err(struct dm_verity *v, enum verity_block_type type, |
245 | unsigned long long block) |
246 | { |
247 | char verity_env[DM_VERITY_ENV_LENGTH]; |
248 | char *envp[] = { verity_env, NULL }; |
249 | const char *type_str = ""; |
250 | struct mapped_device *md = dm_table_get_md(t: v->ti->table); |
251 | |
252 | /* Corruption should be visible in device status in all modes */ |
253 | v->hash_failed = true; |
254 | |
255 | if (v->corrupted_errs >= DM_VERITY_MAX_CORRUPTED_ERRS) |
256 | goto out; |
257 | |
258 | v->corrupted_errs++; |
259 | |
260 | switch (type) { |
261 | case DM_VERITY_BLOCK_TYPE_DATA: |
262 | type_str = "data"; |
263 | break; |
264 | case DM_VERITY_BLOCK_TYPE_METADATA: |
265 | type_str = "metadata"; |
266 | break; |
267 | default: |
268 | BUG(); |
269 | } |
270 | |
271 | DMERR_LIMIT("%s: %s block %llu is corrupted", v->data_dev->name, |
272 | type_str, block); |
273 | |
274 | if (v->corrupted_errs == DM_VERITY_MAX_CORRUPTED_ERRS) { |
275 | DMERR("%s: reached maximum errors", v->data_dev->name); |
276 | dm_audit_log_target(DM_MSG_PREFIX, op: "max-corrupted-errors", ti: v->ti, result: 0); |
277 | } |
278 | |
279 | snprintf(buf: verity_env, DM_VERITY_ENV_LENGTH, fmt: "%s=%d,%llu", |
280 | DM_VERITY_ENV_VAR_NAME, type, block); |
281 | |
282 | kobject_uevent_env(kobj: &disk_to_dev(dm_disk(md))->kobj, action: KOBJ_CHANGE, envp); |
283 | |
284 | out: |
285 | if (v->mode == DM_VERITY_MODE_LOGGING) |
286 | return 0; |
287 | |
288 | if (v->mode == DM_VERITY_MODE_RESTART) |
289 | kernel_restart(cmd: "dm-verity device corrupted"); |
290 | |
291 | if (v->mode == DM_VERITY_MODE_PANIC) |
292 | panic(fmt: "dm-verity device corrupted"); |
293 | |
294 | return 1; |
295 | } |
296 | |
297 | /* |
298 | * Verify hash of a metadata block pertaining to the specified data block |
299 | * ("block" argument) at a specified level ("level" argument). |
300 | * |
301 | * On successful return, verity_io_want_digest(v, io) contains the hash value |
302 | * for a lower tree level or for the data block (if we're at the lowest level). |
303 | * |
304 | * If "skip_unverified" is true, unverified buffer is skipped and 1 is returned. |
305 | * If "skip_unverified" is false, unverified buffer is hashed and verified |
306 | * against current value of verity_io_want_digest(v, io). |
307 | */ |
308 | static int verity_verify_level(struct dm_verity *v, struct dm_verity_io *io, |
309 | sector_t block, int level, bool skip_unverified, |
310 | u8 *want_digest) |
311 | { |
312 | struct dm_buffer *buf; |
313 | struct buffer_aux *aux; |
314 | u8 *data; |
315 | int r; |
316 | sector_t hash_block; |
317 | unsigned int offset; |
318 | struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
319 | |
320 | verity_hash_at_level(v, block, level, hash_block: &hash_block, offset: &offset); |
321 | |
322 | if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { |
323 | data = dm_bufio_get(c: v->bufio, block: hash_block, bp: &buf); |
324 | if (IS_ERR_OR_NULL(ptr: data)) { |
325 | /* |
326 | * In tasklet and the hash was not in the bufio cache. |
327 | * Return early and resume execution from a work-queue |
328 | * to read the hash from disk. |
329 | */ |
330 | return -EAGAIN; |
331 | } |
332 | } else { |
333 | data = dm_bufio_read_with_ioprio(c: v->bufio, block: hash_block, |
334 | bp: &buf, ioprio: bio->bi_ioprio); |
335 | } |
336 | |
337 | if (IS_ERR(ptr: data)) { |
338 | if (skip_unverified) |
339 | return 1; |
340 | r = PTR_ERR(ptr: data); |
341 | data = dm_bufio_new(c: v->bufio, block: hash_block, bp: &buf); |
342 | if (IS_ERR(ptr: data)) |
343 | return r; |
344 | if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA, |
345 | block: hash_block, dest: data) == 0) { |
346 | aux = dm_bufio_get_aux_data(b: buf); |
347 | aux->hash_verified = 1; |
348 | goto release_ok; |
349 | } else { |
350 | dm_bufio_release(b: buf); |
351 | dm_bufio_forget(c: v->bufio, block: hash_block); |
352 | return r; |
353 | } |
354 | } |
355 | |
356 | aux = dm_bufio_get_aux_data(b: buf); |
357 | |
358 | if (!aux->hash_verified) { |
359 | if (skip_unverified) { |
360 | r = 1; |
361 | goto release_ret_r; |
362 | } |
363 | |
364 | r = verity_hash(v, io, data, len: 1 << v->hash_dev_block_bits, |
365 | digest: verity_io_real_digest(v, io), may_sleep: !io->in_bh); |
366 | if (unlikely(r < 0)) |
367 | goto release_ret_r; |
368 | |
369 | if (likely(memcmp(verity_io_real_digest(v, io), want_digest, |
370 | v->digest_size) == 0)) |
371 | aux->hash_verified = 1; |
372 | else if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { |
373 | /* |
374 | * Error handling code (FEC included) cannot be run in a |
375 | * tasklet since it may sleep, so fallback to work-queue. |
376 | */ |
377 | r = -EAGAIN; |
378 | goto release_ret_r; |
379 | } else if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_METADATA, |
380 | block: hash_block, dest: data) == 0) |
381 | aux->hash_verified = 1; |
382 | else if (verity_handle_err(v, |
383 | type: DM_VERITY_BLOCK_TYPE_METADATA, |
384 | block: hash_block)) { |
385 | struct bio *bio; |
386 | io->had_mismatch = true; |
387 | bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
388 | dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-metadata", bio, |
389 | sector: block, result: 0); |
390 | r = -EIO; |
391 | goto release_ret_r; |
392 | } |
393 | } |
394 | |
395 | release_ok: |
396 | data += offset; |
397 | memcpy(want_digest, data, v->digest_size); |
398 | r = 0; |
399 | |
400 | release_ret_r: |
401 | dm_bufio_release(b: buf); |
402 | return r; |
403 | } |
404 | |
405 | /* |
406 | * Find a hash for a given block, write it to digest and verify the integrity |
407 | * of the hash tree if necessary. |
408 | */ |
409 | int verity_hash_for_block(struct dm_verity *v, struct dm_verity_io *io, |
410 | sector_t block, u8 *digest, bool *is_zero) |
411 | { |
412 | int r = 0, i; |
413 | |
414 | if (likely(v->levels)) { |
415 | /* |
416 | * First, we try to get the requested hash for |
417 | * the current block. If the hash block itself is |
418 | * verified, zero is returned. If it isn't, this |
419 | * function returns 1 and we fall back to whole |
420 | * chain verification. |
421 | */ |
422 | r = verity_verify_level(v, io, block, level: 0, skip_unverified: true, want_digest: digest); |
423 | if (likely(r <= 0)) |
424 | goto out; |
425 | } |
426 | |
427 | memcpy(digest, v->root_digest, v->digest_size); |
428 | |
429 | for (i = v->levels - 1; i >= 0; i--) { |
430 | r = verity_verify_level(v, io, block, level: i, skip_unverified: false, want_digest: digest); |
431 | if (unlikely(r)) |
432 | goto out; |
433 | } |
434 | out: |
435 | if (!r && v->zero_digest) |
436 | *is_zero = !memcmp(p: v->zero_digest, q: digest, size: v->digest_size); |
437 | else |
438 | *is_zero = false; |
439 | |
440 | return r; |
441 | } |
442 | |
443 | static noinline int verity_recheck(struct dm_verity *v, struct dm_verity_io *io, |
444 | sector_t cur_block, u8 *dest) |
445 | { |
446 | struct page *page; |
447 | void *buffer; |
448 | int r; |
449 | struct dm_io_request io_req; |
450 | struct dm_io_region io_loc; |
451 | |
452 | page = mempool_alloc(&v->recheck_pool, GFP_NOIO); |
453 | buffer = page_to_virt(page); |
454 | |
455 | io_req.bi_opf = REQ_OP_READ; |
456 | io_req.mem.type = DM_IO_KMEM; |
457 | io_req.mem.ptr.addr = buffer; |
458 | io_req.notify.fn = NULL; |
459 | io_req.client = v->io; |
460 | io_loc.bdev = v->data_dev->bdev; |
461 | io_loc.sector = cur_block << (v->data_dev_block_bits - SECTOR_SHIFT); |
462 | io_loc.count = 1 << (v->data_dev_block_bits - SECTOR_SHIFT); |
463 | r = dm_io(io_req: &io_req, num_regions: 1, region: &io_loc, NULL, IOPRIO_DEFAULT); |
464 | if (unlikely(r)) |
465 | goto free_ret; |
466 | |
467 | r = verity_hash(v, io, data: buffer, len: 1 << v->data_dev_block_bits, |
468 | digest: verity_io_real_digest(v, io), may_sleep: true); |
469 | if (unlikely(r)) |
470 | goto free_ret; |
471 | |
472 | if (memcmp(p: verity_io_real_digest(v, io), |
473 | q: verity_io_want_digest(v, io), size: v->digest_size)) { |
474 | r = -EIO; |
475 | goto free_ret; |
476 | } |
477 | |
478 | memcpy(dest, buffer, 1 << v->data_dev_block_bits); |
479 | r = 0; |
480 | free_ret: |
481 | mempool_free(element: page, pool: &v->recheck_pool); |
482 | |
483 | return r; |
484 | } |
485 | |
486 | static int verity_handle_data_hash_mismatch(struct dm_verity *v, |
487 | struct dm_verity_io *io, |
488 | struct bio *bio, sector_t blkno, |
489 | u8 *data) |
490 | { |
491 | if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { |
492 | /* |
493 | * Error handling code (FEC included) cannot be run in the |
494 | * BH workqueue, so fallback to a standard workqueue. |
495 | */ |
496 | return -EAGAIN; |
497 | } |
498 | if (verity_recheck(v, io, cur_block: blkno, dest: data) == 0) { |
499 | if (v->validated_blocks) |
500 | set_bit(nr: blkno, addr: v->validated_blocks); |
501 | return 0; |
502 | } |
503 | #if defined(CONFIG_DM_VERITY_FEC) |
504 | if (verity_fec_decode(v, io, type: DM_VERITY_BLOCK_TYPE_DATA, block: blkno, |
505 | dest: data) == 0) |
506 | return 0; |
507 | #endif |
508 | if (bio->bi_status) |
509 | return -EIO; /* Error correction failed; Just return error */ |
510 | |
511 | if (verity_handle_err(v, type: DM_VERITY_BLOCK_TYPE_DATA, block: blkno)) { |
512 | io->had_mismatch = true; |
513 | dm_audit_log_bio(DM_MSG_PREFIX, op: "verify-data", bio, sector: blkno, result: 0); |
514 | return -EIO; |
515 | } |
516 | return 0; |
517 | } |
518 | |
519 | /* |
520 | * Verify one "dm_verity_io" structure. |
521 | */ |
522 | static int verity_verify_io(struct dm_verity_io *io) |
523 | { |
524 | struct dm_verity *v = io->v; |
525 | const unsigned int block_size = 1 << v->data_dev_block_bits; |
526 | struct bvec_iter iter_copy; |
527 | struct bvec_iter *iter; |
528 | struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
529 | unsigned int b; |
530 | |
531 | if (static_branch_unlikely(&use_bh_wq_enabled) && io->in_bh) { |
532 | /* |
533 | * Copy the iterator in case we need to restart |
534 | * verification in a work-queue. |
535 | */ |
536 | iter_copy = io->iter; |
537 | iter = &iter_copy; |
538 | } else |
539 | iter = &io->iter; |
540 | |
541 | for (b = 0; b < io->n_blocks; |
542 | b++, bio_advance_iter(bio, iter, bytes: block_size)) { |
543 | int r; |
544 | sector_t cur_block = io->block + b; |
545 | bool is_zero; |
546 | struct bio_vec bv; |
547 | void *data; |
548 | |
549 | if (v->validated_blocks && bio->bi_status == BLK_STS_OK && |
550 | likely(test_bit(cur_block, v->validated_blocks))) |
551 | continue; |
552 | |
553 | r = verity_hash_for_block(v, io, block: cur_block, |
554 | digest: verity_io_want_digest(v, io), |
555 | is_zero: &is_zero); |
556 | if (unlikely(r < 0)) |
557 | return r; |
558 | |
559 | bv = bio_iter_iovec(bio, *iter); |
560 | if (unlikely(bv.bv_len < block_size)) { |
561 | /* |
562 | * Data block spans pages. This should not happen, |
563 | * since dm-verity sets dma_alignment to the data block |
564 | * size minus 1, and dm-verity also doesn't allow the |
565 | * data block size to be greater than PAGE_SIZE. |
566 | */ |
567 | DMERR_LIMIT("unaligned io (data block spans pages)"); |
568 | return -EIO; |
569 | } |
570 | |
571 | data = bvec_kmap_local(bvec: &bv); |
572 | |
573 | if (is_zero) { |
574 | /* |
575 | * If we expect a zero block, don't validate, just |
576 | * return zeros. |
577 | */ |
578 | memset(data, 0, block_size); |
579 | kunmap_local(data); |
580 | continue; |
581 | } |
582 | |
583 | r = verity_hash(v, io, data, len: block_size, |
584 | digest: verity_io_real_digest(v, io), may_sleep: !io->in_bh); |
585 | if (unlikely(r < 0)) { |
586 | kunmap_local(data); |
587 | return r; |
588 | } |
589 | |
590 | if (likely(memcmp(verity_io_real_digest(v, io), |
591 | verity_io_want_digest(v, io), v->digest_size) == 0)) { |
592 | if (v->validated_blocks) |
593 | set_bit(nr: cur_block, addr: v->validated_blocks); |
594 | kunmap_local(data); |
595 | continue; |
596 | } |
597 | r = verity_handle_data_hash_mismatch(v, io, bio, blkno: cur_block, |
598 | data); |
599 | kunmap_local(data); |
600 | if (unlikely(r)) |
601 | return r; |
602 | } |
603 | |
604 | return 0; |
605 | } |
606 | |
607 | /* |
608 | * Skip verity work in response to I/O error when system is shutting down. |
609 | */ |
610 | static inline bool verity_is_system_shutting_down(void) |
611 | { |
612 | return system_state == SYSTEM_HALT || system_state == SYSTEM_POWER_OFF |
613 | || system_state == SYSTEM_RESTART; |
614 | } |
615 | |
616 | static void restart_io_error(struct work_struct *w) |
617 | { |
618 | kernel_restart(cmd: "dm-verity device has I/O error"); |
619 | } |
620 | |
621 | /* |
622 | * End one "io" structure with a given error. |
623 | */ |
624 | static void verity_finish_io(struct dm_verity_io *io, blk_status_t status) |
625 | { |
626 | struct dm_verity *v = io->v; |
627 | struct bio *bio = dm_bio_from_per_bio_data(data: io, data_size: v->ti->per_io_data_size); |
628 | |
629 | bio->bi_end_io = io->orig_bi_end_io; |
630 | bio->bi_status = status; |
631 | |
632 | if (!static_branch_unlikely(&use_bh_wq_enabled) || !io->in_bh) |
633 | verity_fec_finish_io(io); |
634 | |
635 | if (unlikely(status != BLK_STS_OK) && |
636 | unlikely(!(bio->bi_opf & REQ_RAHEAD)) && |
637 | !io->had_mismatch && |
638 | !verity_is_system_shutting_down()) { |
639 | if (v->error_mode == DM_VERITY_MODE_PANIC) { |
640 | panic(fmt: "dm-verity device has I/O error"); |
641 | } |
642 | if (v->error_mode == DM_VERITY_MODE_RESTART) { |
643 | static DECLARE_WORK(restart_work, restart_io_error); |
644 | queue_work(wq: v->verify_wq, work: &restart_work); |
645 | /* |
646 | * We deliberately don't call bio_endio here, because |
647 | * the machine will be restarted anyway. |
648 | */ |
649 | return; |
650 | } |
651 | } |
652 | |
653 | bio_endio(bio); |
654 | } |
655 | |
656 | static void verity_work(struct work_struct *w) |
657 | { |
658 | struct dm_verity_io *io = container_of(w, struct dm_verity_io, work); |
659 | |
660 | io->in_bh = false; |
661 | |
662 | verity_finish_io(io, status: errno_to_blk_status(errno: verity_verify_io(io))); |
663 | } |
664 | |
665 | static void verity_bh_work(struct work_struct *w) |
666 | { |
667 | struct dm_verity_io *io = container_of(w, struct dm_verity_io, bh_work); |
668 | int err; |
669 | |
670 | io->in_bh = true; |
671 | err = verity_verify_io(io); |
672 | if (err == -EAGAIN || err == -ENOMEM) { |
673 | /* fallback to retrying with work-queue */ |
674 | INIT_WORK(&io->work, verity_work); |
675 | queue_work(wq: io->v->verify_wq, work: &io->work); |
676 | return; |
677 | } |
678 | |
679 | verity_finish_io(io, status: errno_to_blk_status(errno: err)); |
680 | } |
681 | |
682 | static inline bool verity_use_bh(unsigned int bytes, unsigned short ioprio) |
683 | { |
684 | return ioprio <= IOPRIO_CLASS_IDLE && |
685 | bytes <= READ_ONCE(dm_verity_use_bh_bytes[ioprio]) && |
686 | !need_resched(); |
687 | } |
688 | |
689 | static void verity_end_io(struct bio *bio) |
690 | { |
691 | struct dm_verity_io *io = bio->bi_private; |
692 | unsigned short ioprio = IOPRIO_PRIO_CLASS(bio->bi_ioprio); |
693 | unsigned int bytes = io->n_blocks << io->v->data_dev_block_bits; |
694 | |
695 | if (bio->bi_status && |
696 | (!verity_fec_is_enabled(v: io->v) || |
697 | verity_is_system_shutting_down() || |
698 | (bio->bi_opf & REQ_RAHEAD))) { |
699 | verity_finish_io(io, status: bio->bi_status); |
700 | return; |
701 | } |
702 | |
703 | if (static_branch_unlikely(&use_bh_wq_enabled) && io->v->use_bh_wq && |
704 | verity_use_bh(bytes, ioprio)) { |
705 | if (in_hardirq() || irqs_disabled()) { |
706 | INIT_WORK(&io->bh_work, verity_bh_work); |
707 | queue_work(wq: system_bh_wq, work: &io->bh_work); |
708 | } else { |
709 | verity_bh_work(w: &io->bh_work); |
710 | } |
711 | } else { |
712 | INIT_WORK(&io->work, verity_work); |
713 | queue_work(wq: io->v->verify_wq, work: &io->work); |
714 | } |
715 | } |
716 | |
717 | /* |
718 | * Prefetch buffers for the specified io. |
719 | * The root buffer is not prefetched, it is assumed that it will be cached |
720 | * all the time. |
721 | */ |
722 | static void verity_prefetch_io(struct work_struct *work) |
723 | { |
724 | struct dm_verity_prefetch_work *pw = |
725 | container_of(work, struct dm_verity_prefetch_work, work); |
726 | struct dm_verity *v = pw->v; |
727 | int i; |
728 | |
729 | for (i = v->levels - 2; i >= 0; i--) { |
730 | sector_t hash_block_start; |
731 | sector_t hash_block_end; |
732 | |
733 | verity_hash_at_level(v, block: pw->block, level: i, hash_block: &hash_block_start, NULL); |
734 | verity_hash_at_level(v, block: pw->block + pw->n_blocks - 1, level: i, hash_block: &hash_block_end, NULL); |
735 | |
736 | if (!i) { |
737 | unsigned int cluster = READ_ONCE(dm_verity_prefetch_cluster); |
738 | |
739 | cluster >>= v->data_dev_block_bits; |
740 | if (unlikely(!cluster)) |
741 | goto no_prefetch_cluster; |
742 | |
743 | if (unlikely(cluster & (cluster - 1))) |
744 | cluster = 1 << __fls(word: cluster); |
745 | |
746 | hash_block_start &= ~(sector_t)(cluster - 1); |
747 | hash_block_end |= cluster - 1; |
748 | if (unlikely(hash_block_end >= v->hash_blocks)) |
749 | hash_block_end = v->hash_blocks - 1; |
750 | } |
751 | no_prefetch_cluster: |
752 | dm_bufio_prefetch_with_ioprio(c: v->bufio, block: hash_block_start, |
753 | n_blocks: hash_block_end - hash_block_start + 1, |
754 | ioprio: pw->ioprio); |
755 | } |
756 | |
757 | kfree(objp: pw); |
758 | } |
759 | |
760 | static void verity_submit_prefetch(struct dm_verity *v, struct dm_verity_io *io, |
761 | unsigned short ioprio) |
762 | { |
763 | sector_t block = io->block; |
764 | unsigned int n_blocks = io->n_blocks; |
765 | struct dm_verity_prefetch_work *pw; |
766 | |
767 | if (v->validated_blocks) { |
768 | while (n_blocks && test_bit(block, v->validated_blocks)) { |
769 | block++; |
770 | n_blocks--; |
771 | } |
772 | while (n_blocks && test_bit(block + n_blocks - 1, |
773 | v->validated_blocks)) |
774 | n_blocks--; |
775 | if (!n_blocks) |
776 | return; |
777 | } |
778 | |
779 | pw = kmalloc(sizeof(struct dm_verity_prefetch_work), |
780 | GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); |
781 | |
782 | if (!pw) |
783 | return; |
784 | |
785 | INIT_WORK(&pw->work, verity_prefetch_io); |
786 | pw->v = v; |
787 | pw->block = block; |
788 | pw->n_blocks = n_blocks; |
789 | pw->ioprio = ioprio; |
790 | queue_work(wq: v->verify_wq, work: &pw->work); |
791 | } |
792 | |
793 | /* |
794 | * Bio map function. It allocates dm_verity_io structure and bio vector and |
795 | * fills them. Then it issues prefetches and the I/O. |
796 | */ |
797 | static int verity_map(struct dm_target *ti, struct bio *bio) |
798 | { |
799 | struct dm_verity *v = ti->private; |
800 | struct dm_verity_io *io; |
801 | |
802 | bio_set_dev(bio, bdev: v->data_dev->bdev); |
803 | bio->bi_iter.bi_sector = verity_map_sector(v, bi_sector: bio->bi_iter.bi_sector); |
804 | |
805 | if (((unsigned int)bio->bi_iter.bi_sector | bio_sectors(bio)) & |
806 | ((1 << (v->data_dev_block_bits - SECTOR_SHIFT)) - 1)) { |
807 | DMERR_LIMIT("unaligned io"); |
808 | return DM_MAPIO_KILL; |
809 | } |
810 | |
811 | if (bio_end_sector(bio) >> |
812 | (v->data_dev_block_bits - SECTOR_SHIFT) > v->data_blocks) { |
813 | DMERR_LIMIT("io out of range"); |
814 | return DM_MAPIO_KILL; |
815 | } |
816 | |
817 | if (bio_data_dir(bio) == WRITE) |
818 | return DM_MAPIO_KILL; |
819 | |
820 | io = dm_per_bio_data(bio, data_size: ti->per_io_data_size); |
821 | io->v = v; |
822 | io->orig_bi_end_io = bio->bi_end_io; |
823 | io->block = bio->bi_iter.bi_sector >> (v->data_dev_block_bits - SECTOR_SHIFT); |
824 | io->n_blocks = bio->bi_iter.bi_size >> v->data_dev_block_bits; |
825 | io->had_mismatch = false; |
826 | |
827 | bio->bi_end_io = verity_end_io; |
828 | bio->bi_private = io; |
829 | io->iter = bio->bi_iter; |
830 | |
831 | verity_fec_init_io(io); |
832 | |
833 | verity_submit_prefetch(v, io, ioprio: bio->bi_ioprio); |
834 | |
835 | submit_bio_noacct(bio); |
836 | |
837 | return DM_MAPIO_SUBMITTED; |
838 | } |
839 | |
840 | static void verity_postsuspend(struct dm_target *ti) |
841 | { |
842 | struct dm_verity *v = ti->private; |
843 | flush_workqueue(v->verify_wq); |
844 | dm_bufio_client_reset(c: v->bufio); |
845 | } |
846 | |
847 | /* |
848 | * Status: V (valid) or C (corruption found) |
849 | */ |
850 | static void verity_status(struct dm_target *ti, status_type_t type, |
851 | unsigned int status_flags, char *result, unsigned int maxlen) |
852 | { |
853 | struct dm_verity *v = ti->private; |
854 | unsigned int args = 0; |
855 | unsigned int sz = 0; |
856 | unsigned int x; |
857 | |
858 | switch (type) { |
859 | case STATUSTYPE_INFO: |
860 | DMEMIT("%c", v->hash_failed ? 'C' : 'V'); |
861 | break; |
862 | case STATUSTYPE_TABLE: |
863 | DMEMIT("%u %s %s %u %u %llu %llu %s ", |
864 | v->version, |
865 | v->data_dev->name, |
866 | v->hash_dev->name, |
867 | 1 << v->data_dev_block_bits, |
868 | 1 << v->hash_dev_block_bits, |
869 | (unsigned long long)v->data_blocks, |
870 | (unsigned long long)v->hash_start, |
871 | v->alg_name |
872 | ); |
873 | for (x = 0; x < v->digest_size; x++) |
874 | DMEMIT("%02x", v->root_digest[x]); |
875 | DMEMIT(" "); |
876 | if (!v->salt_size) |
877 | DMEMIT("-"); |
878 | else |
879 | for (x = 0; x < v->salt_size; x++) |
880 | DMEMIT("%02x", v->salt[x]); |
881 | if (v->mode != DM_VERITY_MODE_EIO) |
882 | args++; |
883 | if (v->error_mode != DM_VERITY_MODE_EIO) |
884 | args++; |
885 | if (verity_fec_is_enabled(v)) |
886 | args += DM_VERITY_OPTS_FEC; |
887 | if (v->zero_digest) |
888 | args++; |
889 | if (v->validated_blocks) |
890 | args++; |
891 | if (v->use_bh_wq) |
892 | args++; |
893 | if (v->signature_key_desc) |
894 | args += DM_VERITY_ROOT_HASH_VERIFICATION_OPTS; |
895 | if (!args) |
896 | return; |
897 | DMEMIT(" %u", args); |
898 | if (v->mode != DM_VERITY_MODE_EIO) { |
899 | DMEMIT(" "); |
900 | switch (v->mode) { |
901 | case DM_VERITY_MODE_LOGGING: |
902 | DMEMIT(DM_VERITY_OPT_LOGGING); |
903 | break; |
904 | case DM_VERITY_MODE_RESTART: |
905 | DMEMIT(DM_VERITY_OPT_RESTART); |
906 | break; |
907 | case DM_VERITY_MODE_PANIC: |
908 | DMEMIT(DM_VERITY_OPT_PANIC); |
909 | break; |
910 | default: |
911 | BUG(); |
912 | } |
913 | } |
914 | if (v->error_mode != DM_VERITY_MODE_EIO) { |
915 | DMEMIT(" "); |
916 | switch (v->error_mode) { |
917 | case DM_VERITY_MODE_RESTART: |
918 | DMEMIT(DM_VERITY_OPT_ERROR_RESTART); |
919 | break; |
920 | case DM_VERITY_MODE_PANIC: |
921 | DMEMIT(DM_VERITY_OPT_ERROR_PANIC); |
922 | break; |
923 | default: |
924 | BUG(); |
925 | } |
926 | } |
927 | if (v->zero_digest) |
928 | DMEMIT(" "DM_VERITY_OPT_IGN_ZEROES); |
929 | if (v->validated_blocks) |
930 | DMEMIT(" "DM_VERITY_OPT_AT_MOST_ONCE); |
931 | if (v->use_bh_wq) |
932 | DMEMIT(" "DM_VERITY_OPT_TASKLET_VERIFY); |
933 | sz = verity_fec_status_table(v, sz, result, maxlen); |
934 | if (v->signature_key_desc) |
935 | DMEMIT(" "DM_VERITY_ROOT_HASH_VERIFICATION_OPT_SIG_KEY |
936 | " %s", v->signature_key_desc); |
937 | break; |
938 | |
939 | case STATUSTYPE_IMA: |
940 | DMEMIT_TARGET_NAME_VERSION(ti->type); |
941 | DMEMIT(",hash_failed=%c", v->hash_failed ? 'C' : 'V'); |
942 | DMEMIT(",verity_version=%u", v->version); |
943 | DMEMIT(",data_device_name=%s", v->data_dev->name); |
944 | DMEMIT(",hash_device_name=%s", v->hash_dev->name); |
945 | DMEMIT(",verity_algorithm=%s", v->alg_name); |
946 | |
947 | DMEMIT(",root_digest="); |
948 | for (x = 0; x < v->digest_size; x++) |
949 | DMEMIT("%02x", v->root_digest[x]); |
950 | |
951 | DMEMIT(",salt="); |
952 | if (!v->salt_size) |
953 | DMEMIT("-"); |
954 | else |
955 | for (x = 0; x < v->salt_size; x++) |
956 | DMEMIT("%02x", v->salt[x]); |
957 | |
958 | DMEMIT(",ignore_zero_blocks=%c", v->zero_digest ? 'y' : 'n'); |
959 | DMEMIT(",check_at_most_once=%c", v->validated_blocks ? 'y' : 'n'); |
960 | if (v->signature_key_desc) |
961 | DMEMIT(",root_hash_sig_key_desc=%s", v->signature_key_desc); |
962 | |
963 | if (v->mode != DM_VERITY_MODE_EIO) { |
964 | DMEMIT(",verity_mode="); |
965 | switch (v->mode) { |
966 | case DM_VERITY_MODE_LOGGING: |
967 | DMEMIT(DM_VERITY_OPT_LOGGING); |
968 | break; |
969 | case DM_VERITY_MODE_RESTART: |
970 | DMEMIT(DM_VERITY_OPT_RESTART); |
971 | break; |
972 | case DM_VERITY_MODE_PANIC: |
973 | DMEMIT(DM_VERITY_OPT_PANIC); |
974 | break; |
975 | default: |
976 | DMEMIT("invalid"); |
977 | } |
978 | } |
979 | if (v->error_mode != DM_VERITY_MODE_EIO) { |
980 | DMEMIT(",verity_error_mode="); |
981 | switch (v->error_mode) { |
982 | case DM_VERITY_MODE_RESTART: |
983 | DMEMIT(DM_VERITY_OPT_ERROR_RESTART); |
984 | break; |
985 | case DM_VERITY_MODE_PANIC: |
986 | DMEMIT(DM_VERITY_OPT_ERROR_PANIC); |
987 | break; |
988 | default: |
989 | DMEMIT("invalid"); |
990 | } |
991 | } |
992 | DMEMIT(";"); |
993 | break; |
994 | } |
995 | } |
996 | |
997 | static int verity_prepare_ioctl(struct dm_target *ti, struct block_device **bdev, |
998 | unsigned int cmd, unsigned long arg, |
999 | bool *forward) |
1000 | { |
1001 | struct dm_verity *v = ti->private; |
1002 | |
1003 | *bdev = v->data_dev->bdev; |
1004 | |
1005 | if (ti->len != bdev_nr_sectors(bdev: v->data_dev->bdev)) |
1006 | return 1; |
1007 | return 0; |
1008 | } |
1009 | |
1010 | static int verity_iterate_devices(struct dm_target *ti, |
1011 | iterate_devices_callout_fn fn, void *data) |
1012 | { |
1013 | struct dm_verity *v = ti->private; |
1014 | |
1015 | return fn(ti, v->data_dev, 0, ti->len, data); |
1016 | } |
1017 | |
1018 | static void verity_io_hints(struct dm_target *ti, struct queue_limits *limits) |
1019 | { |
1020 | struct dm_verity *v = ti->private; |
1021 | |
1022 | if (limits->logical_block_size < 1 << v->data_dev_block_bits) |
1023 | limits->logical_block_size = 1 << v->data_dev_block_bits; |
1024 | |
1025 | if (limits->physical_block_size < 1 << v->data_dev_block_bits) |
1026 | limits->physical_block_size = 1 << v->data_dev_block_bits; |
1027 | |
1028 | limits->io_min = limits->logical_block_size; |
1029 | |
1030 | /* |
1031 | * Similar to what dm-crypt does, opt dm-verity out of support for |
1032 | * direct I/O that is aligned to less than the traditional direct I/O |
1033 | * alignment requirement of logical_block_size. This prevents dm-verity |
1034 | * data blocks from crossing pages, eliminating various edge cases. |
1035 | */ |
1036 | limits->dma_alignment = limits->logical_block_size - 1; |
1037 | } |
1038 | |
1039 | #ifdef CONFIG_SECURITY |
1040 | |
1041 | static int verity_init_sig(struct dm_verity *v, const void *sig, |
1042 | size_t sig_size) |
1043 | { |
1044 | v->sig_size = sig_size; |
1045 | |
1046 | if (sig) { |
1047 | v->root_digest_sig = kmemdup(sig, v->sig_size, GFP_KERNEL); |
1048 | if (!v->root_digest_sig) |
1049 | return -ENOMEM; |
1050 | } |
1051 | |
1052 | return 0; |
1053 | } |
1054 | |
1055 | static void verity_free_sig(struct dm_verity *v) |
1056 | { |
1057 | kfree(objp: v->root_digest_sig); |
1058 | } |
1059 | |
1060 | #else |
1061 | |
1062 | static inline int verity_init_sig(struct dm_verity *v, const void *sig, |
1063 | size_t sig_size) |
1064 | { |
1065 | return 0; |
1066 | } |
1067 | |
1068 | static inline void verity_free_sig(struct dm_verity *v) |
1069 | { |
1070 | } |
1071 | |
1072 | #endif /* CONFIG_SECURITY */ |
1073 | |
1074 | static void verity_dtr(struct dm_target *ti) |
1075 | { |
1076 | struct dm_verity *v = ti->private; |
1077 | |
1078 | if (v->verify_wq) |
1079 | destroy_workqueue(wq: v->verify_wq); |
1080 | |
1081 | mempool_exit(pool: &v->recheck_pool); |
1082 | if (v->io) |
1083 | dm_io_client_destroy(client: v->io); |
1084 | |
1085 | if (v->bufio) |
1086 | dm_bufio_client_destroy(c: v->bufio); |
1087 | |
1088 | kvfree(addr: v->validated_blocks); |
1089 | kfree(objp: v->salt); |
1090 | kfree(objp: v->initial_hashstate); |
1091 | kfree(objp: v->root_digest); |
1092 | kfree(objp: v->zero_digest); |
1093 | verity_free_sig(v); |
1094 | |
1095 | if (v->ahash_tfm) { |
1096 | static_branch_dec(&ahash_enabled); |
1097 | crypto_free_ahash(tfm: v->ahash_tfm); |
1098 | } else { |
1099 | crypto_free_shash(tfm: v->shash_tfm); |
1100 | } |
1101 | |
1102 | kfree(objp: v->alg_name); |
1103 | |
1104 | if (v->hash_dev) |
1105 | dm_put_device(ti, d: v->hash_dev); |
1106 | |
1107 | if (v->data_dev) |
1108 | dm_put_device(ti, d: v->data_dev); |
1109 | |
1110 | verity_fec_dtr(v); |
1111 | |
1112 | kfree(objp: v->signature_key_desc); |
1113 | |
1114 | if (v->use_bh_wq) |
1115 | static_branch_dec(&use_bh_wq_enabled); |
1116 | |
1117 | kfree(objp: v); |
1118 | |
1119 | dm_audit_log_dtr(DM_MSG_PREFIX, ti, result: 1); |
1120 | } |
1121 | |
1122 | static int verity_alloc_most_once(struct dm_verity *v) |
1123 | { |
1124 | struct dm_target *ti = v->ti; |
1125 | |
1126 | if (v->validated_blocks) |
1127 | return 0; |
1128 | |
1129 | /* the bitset can only handle INT_MAX blocks */ |
1130 | if (v->data_blocks > INT_MAX) { |
1131 | ti->error = "device too large to use check_at_most_once"; |
1132 | return -E2BIG; |
1133 | } |
1134 | |
1135 | v->validated_blocks = kvcalloc(BITS_TO_LONGS(v->data_blocks), |
1136 | sizeof(unsigned long), |
1137 | GFP_KERNEL); |
1138 | if (!v->validated_blocks) { |
1139 | ti->error = "failed to allocate bitset for check_at_most_once"; |
1140 | return -ENOMEM; |
1141 | } |
1142 | |
1143 | return 0; |
1144 | } |
1145 | |
1146 | static int verity_alloc_zero_digest(struct dm_verity *v) |
1147 | { |
1148 | int r = -ENOMEM; |
1149 | struct dm_verity_io *io; |
1150 | u8 *zero_data; |
1151 | |
1152 | if (v->zero_digest) |
1153 | return 0; |
1154 | |
1155 | v->zero_digest = kmalloc(v->digest_size, GFP_KERNEL); |
1156 | |
1157 | if (!v->zero_digest) |
1158 | return r; |
1159 | |
1160 | io = kmalloc(sizeof(*io) + v->hash_reqsize, GFP_KERNEL); |
1161 | |
1162 | if (!io) |
1163 | return r; /* verity_dtr will free zero_digest */ |
1164 | |
1165 | zero_data = kzalloc(1 << v->data_dev_block_bits, GFP_KERNEL); |
1166 | |
1167 | if (!zero_data) |
1168 | goto out; |
1169 | |
1170 | r = verity_hash(v, io, data: zero_data, len: 1 << v->data_dev_block_bits, |
1171 | digest: v->zero_digest, may_sleep: true); |
1172 | |
1173 | out: |
1174 | kfree(objp: io); |
1175 | kfree(objp: zero_data); |
1176 | |
1177 | return r; |
1178 | } |
1179 | |
1180 | static inline bool verity_is_verity_mode(const char *arg_name) |
1181 | { |
1182 | return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING) || |
1183 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART) || |
1184 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC)); |
1185 | } |
1186 | |
1187 | static int verity_parse_verity_mode(struct dm_verity *v, const char *arg_name) |
1188 | { |
1189 | if (v->mode) |
1190 | return -EINVAL; |
1191 | |
1192 | if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_LOGGING)) |
1193 | v->mode = DM_VERITY_MODE_LOGGING; |
1194 | else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_RESTART)) |
1195 | v->mode = DM_VERITY_MODE_RESTART; |
1196 | else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_PANIC)) |
1197 | v->mode = DM_VERITY_MODE_PANIC; |
1198 | |
1199 | return 0; |
1200 | } |
1201 | |
1202 | static inline bool verity_is_verity_error_mode(const char *arg_name) |
1203 | { |
1204 | return (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART) || |
1205 | !strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC)); |
1206 | } |
1207 | |
1208 | static int verity_parse_verity_error_mode(struct dm_verity *v, const char *arg_name) |
1209 | { |
1210 | if (v->error_mode) |
1211 | return -EINVAL; |
1212 | |
1213 | if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_RESTART)) |
1214 | v->error_mode = DM_VERITY_MODE_RESTART; |
1215 | else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_ERROR_PANIC)) |
1216 | v->error_mode = DM_VERITY_MODE_PANIC; |
1217 | |
1218 | return 0; |
1219 | } |
1220 | |
1221 | static int verity_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, |
1222 | struct dm_verity_sig_opts *verify_args, |
1223 | bool only_modifier_opts) |
1224 | { |
1225 | int r = 0; |
1226 | unsigned int argc; |
1227 | struct dm_target *ti = v->ti; |
1228 | const char *arg_name; |
1229 | |
1230 | static const struct dm_arg _args[] = { |
1231 | {0, DM_VERITY_OPTS_MAX, "Invalid number of feature args"}, |
1232 | }; |
1233 | |
1234 | r = dm_read_arg_group(arg: _args, arg_set: as, num_args: &argc, error: &ti->error); |
1235 | if (r) |
1236 | return -EINVAL; |
1237 | |
1238 | if (!argc) |
1239 | return 0; |
1240 | |
1241 | do { |
1242 | arg_name = dm_shift_arg(as); |
1243 | argc--; |
1244 | |
1245 | if (verity_is_verity_mode(arg_name)) { |
1246 | if (only_modifier_opts) |
1247 | continue; |
1248 | r = verity_parse_verity_mode(v, arg_name); |
1249 | if (r) { |
1250 | ti->error = "Conflicting error handling parameters"; |
1251 | return r; |
1252 | } |
1253 | continue; |
1254 | |
1255 | } else if (verity_is_verity_error_mode(arg_name)) { |
1256 | if (only_modifier_opts) |
1257 | continue; |
1258 | r = verity_parse_verity_error_mode(v, arg_name); |
1259 | if (r) { |
1260 | ti->error = "Conflicting error handling parameters"; |
1261 | return r; |
1262 | } |
1263 | continue; |
1264 | |
1265 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_IGN_ZEROES)) { |
1266 | if (only_modifier_opts) |
1267 | continue; |
1268 | r = verity_alloc_zero_digest(v); |
1269 | if (r) { |
1270 | ti->error = "Cannot allocate zero digest"; |
1271 | return r; |
1272 | } |
1273 | continue; |
1274 | |
1275 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_AT_MOST_ONCE)) { |
1276 | if (only_modifier_opts) |
1277 | continue; |
1278 | r = verity_alloc_most_once(v); |
1279 | if (r) |
1280 | return r; |
1281 | continue; |
1282 | |
1283 | } else if (!strcasecmp(s1: arg_name, DM_VERITY_OPT_TASKLET_VERIFY)) { |
1284 | v->use_bh_wq = true; |
1285 | static_branch_inc(&use_bh_wq_enabled); |
1286 | continue; |
1287 | |
1288 | } else if (verity_is_fec_opt_arg(arg_name)) { |
1289 | if (only_modifier_opts) |
1290 | continue; |
1291 | r = verity_fec_parse_opt_args(as, v, argc: &argc, arg_name); |
1292 | if (r) |
1293 | return r; |
1294 | continue; |
1295 | |
1296 | } else if (verity_verify_is_sig_opt_arg(arg_name)) { |
1297 | if (only_modifier_opts) |
1298 | continue; |
1299 | r = verity_verify_sig_parse_opt_args(as, v, |
1300 | sig_opts: verify_args, |
1301 | argc: &argc, arg_name); |
1302 | if (r) |
1303 | return r; |
1304 | continue; |
1305 | |
1306 | } else if (only_modifier_opts) { |
1307 | /* |
1308 | * Ignore unrecognized opt, could easily be an extra |
1309 | * argument to an option whose parsing was skipped. |
1310 | * Normal parsing (@only_modifier_opts=false) will |
1311 | * properly parse all options (and their extra args). |
1312 | */ |
1313 | continue; |
1314 | } |
1315 | |
1316 | DMERR("Unrecognized verity feature request: %s", arg_name); |
1317 | ti->error = "Unrecognized verity feature request"; |
1318 | return -EINVAL; |
1319 | } while (argc && !r); |
1320 | |
1321 | return r; |
1322 | } |
1323 | |
1324 | static int verity_setup_hash_alg(struct dm_verity *v, const char *alg_name) |
1325 | { |
1326 | struct dm_target *ti = v->ti; |
1327 | struct crypto_ahash *ahash; |
1328 | struct crypto_shash *shash = NULL; |
1329 | const char *driver_name; |
1330 | |
1331 | v->alg_name = kstrdup(s: alg_name, GFP_KERNEL); |
1332 | if (!v->alg_name) { |
1333 | ti->error = "Cannot allocate algorithm name"; |
1334 | return -ENOMEM; |
1335 | } |
1336 | |
1337 | /* |
1338 | * Allocate the hash transformation object that this dm-verity instance |
1339 | * will use. The vast majority of dm-verity users use CPU-based |
1340 | * hashing, so when possible use the shash API to minimize the crypto |
1341 | * API overhead. If the ahash API resolves to a different driver |
1342 | * (likely an off-CPU hardware offload), use ahash instead. Also use |
1343 | * ahash if the obsolete dm-verity format with the appended salt is |
1344 | * being used, so that quirk only needs to be handled in one place. |
1345 | */ |
1346 | ahash = crypto_alloc_ahash(alg_name, type: 0, |
1347 | mask: v->use_bh_wq ? CRYPTO_ALG_ASYNC : 0); |
1348 | if (IS_ERR(ptr: ahash)) { |
1349 | ti->error = "Cannot initialize hash function"; |
1350 | return PTR_ERR(ptr: ahash); |
1351 | } |
1352 | driver_name = crypto_ahash_driver_name(tfm: ahash); |
1353 | if (v->version >= 1 /* salt prepended, not appended? */) { |
1354 | shash = crypto_alloc_shash(alg_name, type: 0, mask: 0); |
1355 | if (!IS_ERR(ptr: shash) && |
1356 | strcmp(crypto_shash_driver_name(tfm: shash), driver_name) != 0) { |
1357 | /* |
1358 | * ahash gave a different driver than shash, so probably |
1359 | * this is a case of real hardware offload. Use ahash. |
1360 | */ |
1361 | crypto_free_shash(tfm: shash); |
1362 | shash = NULL; |
1363 | } |
1364 | } |
1365 | if (!IS_ERR_OR_NULL(ptr: shash)) { |
1366 | crypto_free_ahash(tfm: ahash); |
1367 | ahash = NULL; |
1368 | v->shash_tfm = shash; |
1369 | v->digest_size = crypto_shash_digestsize(tfm: shash); |
1370 | v->hash_reqsize = sizeof(struct shash_desc) + |
1371 | crypto_shash_descsize(tfm: shash); |
1372 | DMINFO("%s using shash \"%s\"", alg_name, driver_name); |
1373 | } else { |
1374 | v->ahash_tfm = ahash; |
1375 | static_branch_inc(&ahash_enabled); |
1376 | v->digest_size = crypto_ahash_digestsize(tfm: ahash); |
1377 | v->hash_reqsize = sizeof(struct ahash_request) + |
1378 | crypto_ahash_reqsize(tfm: ahash); |
1379 | DMINFO("%s using ahash \"%s\"", alg_name, driver_name); |
1380 | } |
1381 | if ((1 << v->hash_dev_block_bits) < v->digest_size * 2) { |
1382 | ti->error = "Digest size too big"; |
1383 | return -EINVAL; |
1384 | } |
1385 | return 0; |
1386 | } |
1387 | |
1388 | static int verity_setup_salt_and_hashstate(struct dm_verity *v, const char *arg) |
1389 | { |
1390 | struct dm_target *ti = v->ti; |
1391 | |
1392 | if (strcmp(arg, "-") != 0) { |
1393 | v->salt_size = strlen(arg) / 2; |
1394 | v->salt = kmalloc(v->salt_size, GFP_KERNEL); |
1395 | if (!v->salt) { |
1396 | ti->error = "Cannot allocate salt"; |
1397 | return -ENOMEM; |
1398 | } |
1399 | if (strlen(arg) != v->salt_size * 2 || |
1400 | hex2bin(dst: v->salt, src: arg, count: v->salt_size)) { |
1401 | ti->error = "Invalid salt"; |
1402 | return -EINVAL; |
1403 | } |
1404 | } |
1405 | if (v->shash_tfm) { |
1406 | SHASH_DESC_ON_STACK(desc, v->shash_tfm); |
1407 | int r; |
1408 | |
1409 | /* |
1410 | * Compute the pre-salted hash state that can be passed to |
1411 | * crypto_shash_import() for each block later. |
1412 | */ |
1413 | v->initial_hashstate = kmalloc( |
1414 | crypto_shash_statesize(v->shash_tfm), GFP_KERNEL); |
1415 | if (!v->initial_hashstate) { |
1416 | ti->error = "Cannot allocate initial hash state"; |
1417 | return -ENOMEM; |
1418 | } |
1419 | desc->tfm = v->shash_tfm; |
1420 | r = crypto_shash_init(desc) ?: |
1421 | crypto_shash_update(desc, data: v->salt, len: v->salt_size) ?: |
1422 | crypto_shash_export(desc, out: v->initial_hashstate); |
1423 | if (r) { |
1424 | ti->error = "Cannot set up initial hash state"; |
1425 | return r; |
1426 | } |
1427 | } |
1428 | return 0; |
1429 | } |
1430 | |
1431 | /* |
1432 | * Target parameters: |
1433 | * <version> The current format is version 1. |
1434 | * Vsn 0 is compatible with original Chromium OS releases. |
1435 | * <data device> |
1436 | * <hash device> |
1437 | * <data block size> |
1438 | * <hash block size> |
1439 | * <the number of data blocks> |
1440 | * <hash start block> |
1441 | * <algorithm> |
1442 | * <digest> |
1443 | * <salt> Hex string or "-" if no salt. |
1444 | */ |
1445 | static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) |
1446 | { |
1447 | struct dm_verity *v; |
1448 | struct dm_verity_sig_opts verify_args = {0}; |
1449 | struct dm_arg_set as; |
1450 | unsigned int num; |
1451 | unsigned long long num_ll; |
1452 | int r; |
1453 | int i; |
1454 | sector_t hash_position; |
1455 | char dummy; |
1456 | char *root_hash_digest_to_validate; |
1457 | |
1458 | v = kzalloc(sizeof(struct dm_verity), GFP_KERNEL); |
1459 | if (!v) { |
1460 | ti->error = "Cannot allocate verity structure"; |
1461 | return -ENOMEM; |
1462 | } |
1463 | ti->private = v; |
1464 | v->ti = ti; |
1465 | |
1466 | r = verity_fec_ctr_alloc(v); |
1467 | if (r) |
1468 | goto bad; |
1469 | |
1470 | if ((dm_table_get_mode(t: ti->table) & ~BLK_OPEN_READ)) { |
1471 | ti->error = "Device must be readonly"; |
1472 | r = -EINVAL; |
1473 | goto bad; |
1474 | } |
1475 | |
1476 | if (argc < 10) { |
1477 | ti->error = "Not enough arguments"; |
1478 | r = -EINVAL; |
1479 | goto bad; |
1480 | } |
1481 | |
1482 | /* Parse optional parameters that modify primary args */ |
1483 | if (argc > 10) { |
1484 | as.argc = argc - 10; |
1485 | as.argv = argv + 10; |
1486 | r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: true); |
1487 | if (r < 0) |
1488 | goto bad; |
1489 | } |
1490 | |
1491 | if (sscanf(argv[0], "%u%c", &num, &dummy) != 1 || |
1492 | num > 1) { |
1493 | ti->error = "Invalid version"; |
1494 | r = -EINVAL; |
1495 | goto bad; |
1496 | } |
1497 | v->version = num; |
1498 | |
1499 | r = dm_get_device(ti, path: argv[1], BLK_OPEN_READ, result: &v->data_dev); |
1500 | if (r) { |
1501 | ti->error = "Data device lookup failed"; |
1502 | goto bad; |
1503 | } |
1504 | |
1505 | r = dm_get_device(ti, path: argv[2], BLK_OPEN_READ, result: &v->hash_dev); |
1506 | if (r) { |
1507 | ti->error = "Hash device lookup failed"; |
1508 | goto bad; |
1509 | } |
1510 | |
1511 | if (sscanf(argv[3], "%u%c", &num, &dummy) != 1 || |
1512 | !num || (num & (num - 1)) || |
1513 | num < bdev_logical_block_size(bdev: v->data_dev->bdev) || |
1514 | num > PAGE_SIZE) { |
1515 | ti->error = "Invalid data device block size"; |
1516 | r = -EINVAL; |
1517 | goto bad; |
1518 | } |
1519 | v->data_dev_block_bits = __ffs(num); |
1520 | |
1521 | if (sscanf(argv[4], "%u%c", &num, &dummy) != 1 || |
1522 | !num || (num & (num - 1)) || |
1523 | num < bdev_logical_block_size(bdev: v->hash_dev->bdev) || |
1524 | num > INT_MAX) { |
1525 | ti->error = "Invalid hash device block size"; |
1526 | r = -EINVAL; |
1527 | goto bad; |
1528 | } |
1529 | v->hash_dev_block_bits = __ffs(num); |
1530 | |
1531 | if (sscanf(argv[5], "%llu%c", &num_ll, &dummy) != 1 || |
1532 | (sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) |
1533 | >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll) { |
1534 | ti->error = "Invalid data blocks"; |
1535 | r = -EINVAL; |
1536 | goto bad; |
1537 | } |
1538 | v->data_blocks = num_ll; |
1539 | |
1540 | if (ti->len > (v->data_blocks << (v->data_dev_block_bits - SECTOR_SHIFT))) { |
1541 | ti->error = "Data device is too small"; |
1542 | r = -EINVAL; |
1543 | goto bad; |
1544 | } |
1545 | |
1546 | if (sscanf(argv[6], "%llu%c", &num_ll, &dummy) != 1 || |
1547 | (sector_t)(num_ll << (v->hash_dev_block_bits - SECTOR_SHIFT)) |
1548 | >> (v->hash_dev_block_bits - SECTOR_SHIFT) != num_ll) { |
1549 | ti->error = "Invalid hash start"; |
1550 | r = -EINVAL; |
1551 | goto bad; |
1552 | } |
1553 | v->hash_start = num_ll; |
1554 | |
1555 | r = verity_setup_hash_alg(v, alg_name: argv[7]); |
1556 | if (r) |
1557 | goto bad; |
1558 | |
1559 | v->root_digest = kmalloc(v->digest_size, GFP_KERNEL); |
1560 | if (!v->root_digest) { |
1561 | ti->error = "Cannot allocate root digest"; |
1562 | r = -ENOMEM; |
1563 | goto bad; |
1564 | } |
1565 | if (strlen(argv[8]) != v->digest_size * 2 || |
1566 | hex2bin(dst: v->root_digest, src: argv[8], count: v->digest_size)) { |
1567 | ti->error = "Invalid root digest"; |
1568 | r = -EINVAL; |
1569 | goto bad; |
1570 | } |
1571 | root_hash_digest_to_validate = argv[8]; |
1572 | |
1573 | r = verity_setup_salt_and_hashstate(v, arg: argv[9]); |
1574 | if (r) |
1575 | goto bad; |
1576 | |
1577 | argv += 10; |
1578 | argc -= 10; |
1579 | |
1580 | /* Optional parameters */ |
1581 | if (argc) { |
1582 | as.argc = argc; |
1583 | as.argv = argv; |
1584 | r = verity_parse_opt_args(as: &as, v, verify_args: &verify_args, only_modifier_opts: false); |
1585 | if (r < 0) |
1586 | goto bad; |
1587 | } |
1588 | |
1589 | /* Root hash signature is an optional parameter */ |
1590 | r = verity_verify_root_hash(data: root_hash_digest_to_validate, |
1591 | strlen(root_hash_digest_to_validate), |
1592 | sig_data: verify_args.sig, |
1593 | sig_len: verify_args.sig_size); |
1594 | if (r < 0) { |
1595 | ti->error = "Root hash verification failed"; |
1596 | goto bad; |
1597 | } |
1598 | |
1599 | r = verity_init_sig(v, sig: verify_args.sig, sig_size: verify_args.sig_size); |
1600 | if (r < 0) { |
1601 | ti->error = "Cannot allocate root digest signature"; |
1602 | goto bad; |
1603 | } |
1604 | |
1605 | v->hash_per_block_bits = |
1606 | __fls(word: (1 << v->hash_dev_block_bits) / v->digest_size); |
1607 | |
1608 | v->levels = 0; |
1609 | if (v->data_blocks) |
1610 | while (v->hash_per_block_bits * v->levels < 64 && |
1611 | (unsigned long long)(v->data_blocks - 1) >> |
1612 | (v->hash_per_block_bits * v->levels)) |
1613 | v->levels++; |
1614 | |
1615 | if (v->levels > DM_VERITY_MAX_LEVELS) { |
1616 | ti->error = "Too many tree levels"; |
1617 | r = -E2BIG; |
1618 | goto bad; |
1619 | } |
1620 | |
1621 | hash_position = v->hash_start; |
1622 | for (i = v->levels - 1; i >= 0; i--) { |
1623 | sector_t s; |
1624 | |
1625 | v->hash_level_block[i] = hash_position; |
1626 | s = (v->data_blocks + ((sector_t)1 << ((i + 1) * v->hash_per_block_bits)) - 1) |
1627 | >> ((i + 1) * v->hash_per_block_bits); |
1628 | if (hash_position + s < hash_position) { |
1629 | ti->error = "Hash device offset overflow"; |
1630 | r = -E2BIG; |
1631 | goto bad; |
1632 | } |
1633 | hash_position += s; |
1634 | } |
1635 | v->hash_blocks = hash_position; |
1636 | |
1637 | r = mempool_init_page_pool(&v->recheck_pool, 1, 0); |
1638 | if (unlikely(r)) { |
1639 | ti->error = "Cannot allocate mempool"; |
1640 | goto bad; |
1641 | } |
1642 | |
1643 | v->io = dm_io_client_create(); |
1644 | if (IS_ERR(ptr: v->io)) { |
1645 | r = PTR_ERR(ptr: v->io); |
1646 | v->io = NULL; |
1647 | ti->error = "Cannot allocate dm io"; |
1648 | goto bad; |
1649 | } |
1650 | |
1651 | v->bufio = dm_bufio_client_create(bdev: v->hash_dev->bdev, |
1652 | block_size: 1 << v->hash_dev_block_bits, reserved_buffers: 1, aux_size: sizeof(struct buffer_aux), |
1653 | alloc_callback: dm_bufio_alloc_callback, NULL, |
1654 | flags: v->use_bh_wq ? DM_BUFIO_CLIENT_NO_SLEEP : 0); |
1655 | if (IS_ERR(ptr: v->bufio)) { |
1656 | ti->error = "Cannot initialize dm-bufio"; |
1657 | r = PTR_ERR(ptr: v->bufio); |
1658 | v->bufio = NULL; |
1659 | goto bad; |
1660 | } |
1661 | |
1662 | if (dm_bufio_get_device_size(c: v->bufio) < v->hash_blocks) { |
1663 | ti->error = "Hash device is too small"; |
1664 | r = -E2BIG; |
1665 | goto bad; |
1666 | } |
1667 | |
1668 | /* |
1669 | * Using WQ_HIGHPRI improves throughput and completion latency by |
1670 | * reducing wait times when reading from a dm-verity device. |
1671 | * |
1672 | * Also as required for the "try_verify_in_tasklet" feature: WQ_HIGHPRI |
1673 | * allows verify_wq to preempt softirq since verification in BH workqueue |
1674 | * will fall-back to using it for error handling (or if the bufio cache |
1675 | * doesn't have required hashes). |
1676 | */ |
1677 | v->verify_wq = alloc_workqueue(fmt: "kverityd", flags: WQ_MEM_RECLAIM | WQ_HIGHPRI, max_active: 0); |
1678 | if (!v->verify_wq) { |
1679 | ti->error = "Cannot allocate workqueue"; |
1680 | r = -ENOMEM; |
1681 | goto bad; |
1682 | } |
1683 | |
1684 | ti->per_io_data_size = sizeof(struct dm_verity_io) + v->hash_reqsize; |
1685 | |
1686 | r = verity_fec_ctr(v); |
1687 | if (r) |
1688 | goto bad; |
1689 | |
1690 | ti->per_io_data_size = roundup(ti->per_io_data_size, |
1691 | __alignof__(struct dm_verity_io)); |
1692 | |
1693 | verity_verify_sig_opts_cleanup(sig_opts: &verify_args); |
1694 | |
1695 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 1); |
1696 | |
1697 | return 0; |
1698 | |
1699 | bad: |
1700 | |
1701 | verity_verify_sig_opts_cleanup(sig_opts: &verify_args); |
1702 | dm_audit_log_ctr(DM_MSG_PREFIX, ti, result: 0); |
1703 | verity_dtr(ti); |
1704 | |
1705 | return r; |
1706 | } |
1707 | |
1708 | /* |
1709 | * Get the verity mode (error behavior) of a verity target. |
1710 | * |
1711 | * Returns the verity mode of the target, or -EINVAL if 'ti' is not a verity |
1712 | * target. |
1713 | */ |
1714 | int dm_verity_get_mode(struct dm_target *ti) |
1715 | { |
1716 | struct dm_verity *v = ti->private; |
1717 | |
1718 | if (!dm_is_verity_target(ti)) |
1719 | return -EINVAL; |
1720 | |
1721 | return v->mode; |
1722 | } |
1723 | |
1724 | /* |
1725 | * Get the root digest of a verity target. |
1726 | * |
1727 | * Returns a copy of the root digest, the caller is responsible for |
1728 | * freeing the memory of the digest. |
1729 | */ |
1730 | int dm_verity_get_root_digest(struct dm_target *ti, u8 **root_digest, unsigned int *digest_size) |
1731 | { |
1732 | struct dm_verity *v = ti->private; |
1733 | |
1734 | if (!dm_is_verity_target(ti)) |
1735 | return -EINVAL; |
1736 | |
1737 | *root_digest = kmemdup(v->root_digest, v->digest_size, GFP_KERNEL); |
1738 | if (*root_digest == NULL) |
1739 | return -ENOMEM; |
1740 | |
1741 | *digest_size = v->digest_size; |
1742 | |
1743 | return 0; |
1744 | } |
1745 | |
1746 | #ifdef CONFIG_SECURITY |
1747 | |
1748 | #ifdef CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG |
1749 | |
1750 | static int verity_security_set_signature(struct block_device *bdev, |
1751 | struct dm_verity *v) |
1752 | { |
1753 | /* |
1754 | * if the dm-verity target is unsigned, v->root_digest_sig will |
1755 | * be NULL, and the hook call is still required to let LSMs mark |
1756 | * the device as unsigned. This information is crucial for LSMs to |
1757 | * block operations such as execution on unsigned files |
1758 | */ |
1759 | return security_bdev_setintegrity(bdev, |
1760 | type: LSM_INT_DMVERITY_SIG_VALID, |
1761 | value: v->root_digest_sig, |
1762 | size: v->sig_size); |
1763 | } |
1764 | |
1765 | #else |
1766 | |
1767 | static inline int verity_security_set_signature(struct block_device *bdev, |
1768 | struct dm_verity *v) |
1769 | { |
1770 | return 0; |
1771 | } |
1772 | |
1773 | #endif /* CONFIG_DM_VERITY_VERIFY_ROOTHASH_SIG */ |
1774 | |
1775 | /* |
1776 | * Expose verity target's root hash and signature data to LSMs before resume. |
1777 | * |
1778 | * Returns 0 on success, or -ENOMEM if the system is out of memory. |
1779 | */ |
1780 | static int verity_preresume(struct dm_target *ti) |
1781 | { |
1782 | struct block_device *bdev; |
1783 | struct dm_verity_digest root_digest; |
1784 | struct dm_verity *v; |
1785 | int r; |
1786 | |
1787 | v = ti->private; |
1788 | bdev = dm_disk(md: dm_table_get_md(t: ti->table))->part0; |
1789 | root_digest.digest = v->root_digest; |
1790 | root_digest.digest_len = v->digest_size; |
1791 | if (static_branch_unlikely(&ahash_enabled) && !v->shash_tfm) |
1792 | root_digest.alg = crypto_ahash_alg_name(tfm: v->ahash_tfm); |
1793 | else |
1794 | root_digest.alg = crypto_shash_alg_name(tfm: v->shash_tfm); |
1795 | |
1796 | r = security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, value: &root_digest, |
1797 | size: sizeof(root_digest)); |
1798 | if (r) |
1799 | return r; |
1800 | |
1801 | r = verity_security_set_signature(bdev, v); |
1802 | if (r) |
1803 | goto bad; |
1804 | |
1805 | return 0; |
1806 | |
1807 | bad: |
1808 | |
1809 | security_bdev_setintegrity(bdev, type: LSM_INT_DMVERITY_ROOTHASH, NULL, size: 0); |
1810 | |
1811 | return r; |
1812 | } |
1813 | |
1814 | #endif /* CONFIG_SECURITY */ |
1815 | |
1816 | static struct target_type verity_target = { |
1817 | .name = "verity", |
1818 | /* Note: the LSMs depend on the singleton and immutable features */ |
1819 | .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE, |
1820 | .version = {1, 11, 0}, |
1821 | .module = THIS_MODULE, |
1822 | .ctr = verity_ctr, |
1823 | .dtr = verity_dtr, |
1824 | .map = verity_map, |
1825 | .postsuspend = verity_postsuspend, |
1826 | .status = verity_status, |
1827 | .prepare_ioctl = verity_prepare_ioctl, |
1828 | .iterate_devices = verity_iterate_devices, |
1829 | .io_hints = verity_io_hints, |
1830 | #ifdef CONFIG_SECURITY |
1831 | .preresume = verity_preresume, |
1832 | #endif /* CONFIG_SECURITY */ |
1833 | }; |
1834 | module_dm(verity); |
1835 | |
1836 | /* |
1837 | * Check whether a DM target is a verity target. |
1838 | */ |
1839 | bool dm_is_verity_target(struct dm_target *ti) |
1840 | { |
1841 | return ti->type == &verity_target; |
1842 | } |
1843 | |
1844 | MODULE_AUTHOR("Mikulas Patocka <mpatocka@redhat.com>"); |
1845 | MODULE_AUTHOR("Mandeep Baines <msb@chromium.org>"); |
1846 | MODULE_AUTHOR("Will Drewry <wad@chromium.org>"); |
1847 | MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); |
1848 | MODULE_LICENSE("GPL"); |
1849 |
Definitions
- dm_verity_prefetch_cluster
- dm_verity_use_bh_bytes
- use_bh_wq_enabled
- ahash_enabled
- dm_verity_prefetch_work
- buffer_aux
- dm_bufio_alloc_callback
- verity_map_sector
- verity_position_at_level
- verity_ahash_update
- verity_ahash_init
- verity_ahash_final
- verity_hash
- verity_hash_at_level
- verity_handle_err
- verity_verify_level
- verity_hash_for_block
- verity_recheck
- verity_handle_data_hash_mismatch
- verity_verify_io
- verity_is_system_shutting_down
- restart_io_error
- verity_finish_io
- verity_work
- verity_bh_work
- verity_use_bh
- verity_end_io
- verity_prefetch_io
- verity_submit_prefetch
- verity_map
- verity_postsuspend
- verity_status
- verity_prepare_ioctl
- verity_iterate_devices
- verity_io_hints
- verity_init_sig
- verity_free_sig
- verity_dtr
- verity_alloc_most_once
- verity_alloc_zero_digest
- verity_is_verity_mode
- verity_parse_verity_mode
- verity_is_verity_error_mode
- verity_parse_verity_error_mode
- verity_parse_opt_args
- verity_setup_hash_alg
- verity_setup_salt_and_hashstate
- verity_ctr
- dm_verity_get_mode
- dm_verity_get_root_digest
- verity_security_set_signature
- verity_preresume
- verity_target
Improve your Profiling and Debugging skills
Find out more