1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright 2019 Google LLC |
4 | */ |
5 | |
6 | /** |
7 | * DOC: blk-crypto profiles |
8 | * |
9 | * 'struct blk_crypto_profile' contains all generic inline encryption-related |
10 | * state for a particular inline encryption device. blk_crypto_profile serves |
11 | * as the way that drivers for inline encryption hardware expose their crypto |
12 | * capabilities and certain functions (e.g., functions to program and evict |
13 | * keys) to upper layers. Device drivers that want to support inline encryption |
14 | * construct a crypto profile, then associate it with the disk's request_queue. |
15 | * |
16 | * If the device has keyslots, then its blk_crypto_profile also handles managing |
17 | * these keyslots in a device-independent way, using the driver-provided |
18 | * functions to program and evict keys as needed. This includes keeping track |
19 | * of which key and how many I/O requests are using each keyslot, getting |
20 | * keyslots for I/O requests, and handling key eviction requests. |
21 | * |
22 | * For more information, see Documentation/block/inline-encryption.rst. |
23 | */ |
24 | |
25 | #define pr_fmt(fmt) "blk-crypto: " fmt |
26 | |
27 | #include <linux/blk-crypto-profile.h> |
28 | #include <linux/device.h> |
29 | #include <linux/atomic.h> |
30 | #include <linux/mutex.h> |
31 | #include <linux/pm_runtime.h> |
32 | #include <linux/wait.h> |
33 | #include <linux/blkdev.h> |
34 | #include <linux/blk-integrity.h> |
35 | #include "blk-crypto-internal.h" |
36 | |
37 | struct blk_crypto_keyslot { |
38 | atomic_t slot_refs; |
39 | struct list_head idle_slot_node; |
40 | struct hlist_node hash_node; |
41 | const struct blk_crypto_key *key; |
42 | struct blk_crypto_profile *profile; |
43 | }; |
44 | |
45 | static inline void blk_crypto_hw_enter(struct blk_crypto_profile *profile) |
46 | { |
47 | /* |
48 | * Calling into the driver requires profile->lock held and the device |
49 | * resumed. But we must resume the device first, since that can acquire |
50 | * and release profile->lock via blk_crypto_reprogram_all_keys(). |
51 | */ |
52 | if (profile->dev) |
53 | pm_runtime_get_sync(dev: profile->dev); |
54 | down_write(sem: &profile->lock); |
55 | } |
56 | |
57 | static inline void blk_crypto_hw_exit(struct blk_crypto_profile *profile) |
58 | { |
59 | up_write(sem: &profile->lock); |
60 | if (profile->dev) |
61 | pm_runtime_put_sync(dev: profile->dev); |
62 | } |
63 | |
64 | /** |
65 | * blk_crypto_profile_init() - Initialize a blk_crypto_profile |
66 | * @profile: the blk_crypto_profile to initialize |
67 | * @num_slots: the number of keyslots |
68 | * |
69 | * Storage drivers must call this when starting to set up a blk_crypto_profile, |
70 | * before filling in additional fields. |
71 | * |
72 | * Return: 0 on success, or else a negative error code. |
73 | */ |
74 | int blk_crypto_profile_init(struct blk_crypto_profile *profile, |
75 | unsigned int num_slots) |
76 | { |
77 | unsigned int slot; |
78 | unsigned int i; |
79 | unsigned int slot_hashtable_size; |
80 | |
81 | memset(profile, 0, sizeof(*profile)); |
82 | |
83 | /* |
84 | * profile->lock of an underlying device can nest inside profile->lock |
85 | * of a device-mapper device, so use a dynamic lock class to avoid |
86 | * false-positive lockdep reports. |
87 | */ |
88 | lockdep_register_key(key: &profile->lockdep_key); |
89 | __init_rwsem(sem: &profile->lock, name: "&profile->lock" , key: &profile->lockdep_key); |
90 | |
91 | if (num_slots == 0) |
92 | return 0; |
93 | |
94 | /* Initialize keyslot management data. */ |
95 | |
96 | profile->slots = kvcalloc(n: num_slots, size: sizeof(profile->slots[0]), |
97 | GFP_KERNEL); |
98 | if (!profile->slots) |
99 | goto err_destroy; |
100 | |
101 | profile->num_slots = num_slots; |
102 | |
103 | init_waitqueue_head(&profile->idle_slots_wait_queue); |
104 | INIT_LIST_HEAD(list: &profile->idle_slots); |
105 | |
106 | for (slot = 0; slot < num_slots; slot++) { |
107 | profile->slots[slot].profile = profile; |
108 | list_add_tail(new: &profile->slots[slot].idle_slot_node, |
109 | head: &profile->idle_slots); |
110 | } |
111 | |
112 | spin_lock_init(&profile->idle_slots_lock); |
113 | |
114 | slot_hashtable_size = roundup_pow_of_two(num_slots); |
115 | /* |
116 | * hash_ptr() assumes bits != 0, so ensure the hash table has at least 2 |
117 | * buckets. This only makes a difference when there is only 1 keyslot. |
118 | */ |
119 | if (slot_hashtable_size < 2) |
120 | slot_hashtable_size = 2; |
121 | |
122 | profile->log_slot_ht_size = ilog2(slot_hashtable_size); |
123 | profile->slot_hashtable = |
124 | kvmalloc_array(n: slot_hashtable_size, |
125 | size: sizeof(profile->slot_hashtable[0]), GFP_KERNEL); |
126 | if (!profile->slot_hashtable) |
127 | goto err_destroy; |
128 | for (i = 0; i < slot_hashtable_size; i++) |
129 | INIT_HLIST_HEAD(&profile->slot_hashtable[i]); |
130 | |
131 | return 0; |
132 | |
133 | err_destroy: |
134 | blk_crypto_profile_destroy(profile); |
135 | return -ENOMEM; |
136 | } |
137 | EXPORT_SYMBOL_GPL(blk_crypto_profile_init); |
138 | |
139 | static void blk_crypto_profile_destroy_callback(void *profile) |
140 | { |
141 | blk_crypto_profile_destroy(profile); |
142 | } |
143 | |
144 | /** |
145 | * devm_blk_crypto_profile_init() - Resource-managed blk_crypto_profile_init() |
146 | * @dev: the device which owns the blk_crypto_profile |
147 | * @profile: the blk_crypto_profile to initialize |
148 | * @num_slots: the number of keyslots |
149 | * |
150 | * Like blk_crypto_profile_init(), but causes blk_crypto_profile_destroy() to be |
151 | * called automatically on driver detach. |
152 | * |
153 | * Return: 0 on success, or else a negative error code. |
154 | */ |
155 | int devm_blk_crypto_profile_init(struct device *dev, |
156 | struct blk_crypto_profile *profile, |
157 | unsigned int num_slots) |
158 | { |
159 | int err = blk_crypto_profile_init(profile, num_slots); |
160 | |
161 | if (err) |
162 | return err; |
163 | |
164 | return devm_add_action_or_reset(dev, |
165 | blk_crypto_profile_destroy_callback, |
166 | profile); |
167 | } |
168 | EXPORT_SYMBOL_GPL(devm_blk_crypto_profile_init); |
169 | |
170 | static inline struct hlist_head * |
171 | blk_crypto_hash_bucket_for_key(struct blk_crypto_profile *profile, |
172 | const struct blk_crypto_key *key) |
173 | { |
174 | return &profile->slot_hashtable[ |
175 | hash_ptr(ptr: key, bits: profile->log_slot_ht_size)]; |
176 | } |
177 | |
178 | static void |
179 | blk_crypto_remove_slot_from_lru_list(struct blk_crypto_keyslot *slot) |
180 | { |
181 | struct blk_crypto_profile *profile = slot->profile; |
182 | unsigned long flags; |
183 | |
184 | spin_lock_irqsave(&profile->idle_slots_lock, flags); |
185 | list_del(entry: &slot->idle_slot_node); |
186 | spin_unlock_irqrestore(lock: &profile->idle_slots_lock, flags); |
187 | } |
188 | |
189 | static struct blk_crypto_keyslot * |
190 | blk_crypto_find_keyslot(struct blk_crypto_profile *profile, |
191 | const struct blk_crypto_key *key) |
192 | { |
193 | const struct hlist_head *head = |
194 | blk_crypto_hash_bucket_for_key(profile, key); |
195 | struct blk_crypto_keyslot *slotp; |
196 | |
197 | hlist_for_each_entry(slotp, head, hash_node) { |
198 | if (slotp->key == key) |
199 | return slotp; |
200 | } |
201 | return NULL; |
202 | } |
203 | |
204 | static struct blk_crypto_keyslot * |
205 | blk_crypto_find_and_grab_keyslot(struct blk_crypto_profile *profile, |
206 | const struct blk_crypto_key *key) |
207 | { |
208 | struct blk_crypto_keyslot *slot; |
209 | |
210 | slot = blk_crypto_find_keyslot(profile, key); |
211 | if (!slot) |
212 | return NULL; |
213 | if (atomic_inc_return(v: &slot->slot_refs) == 1) { |
214 | /* Took first reference to this slot; remove it from LRU list */ |
215 | blk_crypto_remove_slot_from_lru_list(slot); |
216 | } |
217 | return slot; |
218 | } |
219 | |
220 | /** |
221 | * blk_crypto_keyslot_index() - Get the index of a keyslot |
222 | * @slot: a keyslot that blk_crypto_get_keyslot() returned |
223 | * |
224 | * Return: the 0-based index of the keyslot within the device's keyslots. |
225 | */ |
226 | unsigned int blk_crypto_keyslot_index(struct blk_crypto_keyslot *slot) |
227 | { |
228 | return slot - slot->profile->slots; |
229 | } |
230 | EXPORT_SYMBOL_GPL(blk_crypto_keyslot_index); |
231 | |
232 | /** |
233 | * blk_crypto_get_keyslot() - Get a keyslot for a key, if needed. |
234 | * @profile: the crypto profile of the device the key will be used on |
235 | * @key: the key that will be used |
236 | * @slot_ptr: If a keyslot is allocated, an opaque pointer to the keyslot struct |
237 | * will be stored here. blk_crypto_put_keyslot() must be called |
238 | * later to release it. Otherwise, NULL will be stored here. |
239 | * |
240 | * If the device has keyslots, this gets a keyslot that's been programmed with |
241 | * the specified key. If the key is already in a slot, this reuses it; |
242 | * otherwise this waits for a slot to become idle and programs the key into it. |
243 | * |
244 | * Context: Process context. Takes and releases profile->lock. |
245 | * Return: BLK_STS_OK on success, meaning that either a keyslot was allocated or |
246 | * one wasn't needed; or a blk_status_t error on failure. |
247 | */ |
248 | blk_status_t blk_crypto_get_keyslot(struct blk_crypto_profile *profile, |
249 | const struct blk_crypto_key *key, |
250 | struct blk_crypto_keyslot **slot_ptr) |
251 | { |
252 | struct blk_crypto_keyslot *slot; |
253 | int slot_idx; |
254 | int err; |
255 | |
256 | *slot_ptr = NULL; |
257 | |
258 | /* |
259 | * If the device has no concept of "keyslots", then there is no need to |
260 | * get one. |
261 | */ |
262 | if (profile->num_slots == 0) |
263 | return BLK_STS_OK; |
264 | |
265 | down_read(sem: &profile->lock); |
266 | slot = blk_crypto_find_and_grab_keyslot(profile, key); |
267 | up_read(sem: &profile->lock); |
268 | if (slot) |
269 | goto success; |
270 | |
271 | for (;;) { |
272 | blk_crypto_hw_enter(profile); |
273 | slot = blk_crypto_find_and_grab_keyslot(profile, key); |
274 | if (slot) { |
275 | blk_crypto_hw_exit(profile); |
276 | goto success; |
277 | } |
278 | |
279 | /* |
280 | * If we're here, that means there wasn't a slot that was |
281 | * already programmed with the key. So try to program it. |
282 | */ |
283 | if (!list_empty(head: &profile->idle_slots)) |
284 | break; |
285 | |
286 | blk_crypto_hw_exit(profile); |
287 | wait_event(profile->idle_slots_wait_queue, |
288 | !list_empty(&profile->idle_slots)); |
289 | } |
290 | |
291 | slot = list_first_entry(&profile->idle_slots, struct blk_crypto_keyslot, |
292 | idle_slot_node); |
293 | slot_idx = blk_crypto_keyslot_index(slot); |
294 | |
295 | err = profile->ll_ops.keyslot_program(profile, key, slot_idx); |
296 | if (err) { |
297 | wake_up(&profile->idle_slots_wait_queue); |
298 | blk_crypto_hw_exit(profile); |
299 | return errno_to_blk_status(errno: err); |
300 | } |
301 | |
302 | /* Move this slot to the hash list for the new key. */ |
303 | if (slot->key) |
304 | hlist_del(n: &slot->hash_node); |
305 | slot->key = key; |
306 | hlist_add_head(n: &slot->hash_node, |
307 | h: blk_crypto_hash_bucket_for_key(profile, key)); |
308 | |
309 | atomic_set(v: &slot->slot_refs, i: 1); |
310 | |
311 | blk_crypto_remove_slot_from_lru_list(slot); |
312 | |
313 | blk_crypto_hw_exit(profile); |
314 | success: |
315 | *slot_ptr = slot; |
316 | return BLK_STS_OK; |
317 | } |
318 | |
319 | /** |
320 | * blk_crypto_put_keyslot() - Release a reference to a keyslot |
321 | * @slot: The keyslot to release the reference of |
322 | * |
323 | * Context: Any context. |
324 | */ |
325 | void blk_crypto_put_keyslot(struct blk_crypto_keyslot *slot) |
326 | { |
327 | struct blk_crypto_profile *profile = slot->profile; |
328 | unsigned long flags; |
329 | |
330 | if (atomic_dec_and_lock_irqsave(&slot->slot_refs, |
331 | &profile->idle_slots_lock, flags)) { |
332 | list_add_tail(new: &slot->idle_slot_node, head: &profile->idle_slots); |
333 | spin_unlock_irqrestore(lock: &profile->idle_slots_lock, flags); |
334 | wake_up(&profile->idle_slots_wait_queue); |
335 | } |
336 | } |
337 | |
338 | /** |
339 | * __blk_crypto_cfg_supported() - Check whether the given crypto profile |
340 | * supports the given crypto configuration. |
341 | * @profile: the crypto profile to check |
342 | * @cfg: the crypto configuration to check for |
343 | * |
344 | * Return: %true if @profile supports the given @cfg. |
345 | */ |
346 | bool __blk_crypto_cfg_supported(struct blk_crypto_profile *profile, |
347 | const struct blk_crypto_config *cfg) |
348 | { |
349 | if (!profile) |
350 | return false; |
351 | if (!(profile->modes_supported[cfg->crypto_mode] & cfg->data_unit_size)) |
352 | return false; |
353 | if (profile->max_dun_bytes_supported < cfg->dun_bytes) |
354 | return false; |
355 | return true; |
356 | } |
357 | |
358 | /* |
359 | * This is an internal function that evicts a key from an inline encryption |
360 | * device that can be either a real device or the blk-crypto-fallback "device". |
361 | * It is used only by blk_crypto_evict_key(); see that function for details. |
362 | */ |
363 | int __blk_crypto_evict_key(struct blk_crypto_profile *profile, |
364 | const struct blk_crypto_key *key) |
365 | { |
366 | struct blk_crypto_keyslot *slot; |
367 | int err; |
368 | |
369 | if (profile->num_slots == 0) { |
370 | if (profile->ll_ops.keyslot_evict) { |
371 | blk_crypto_hw_enter(profile); |
372 | err = profile->ll_ops.keyslot_evict(profile, key, -1); |
373 | blk_crypto_hw_exit(profile); |
374 | return err; |
375 | } |
376 | return 0; |
377 | } |
378 | |
379 | blk_crypto_hw_enter(profile); |
380 | slot = blk_crypto_find_keyslot(profile, key); |
381 | if (!slot) { |
382 | /* |
383 | * Not an error, since a key not in use by I/O is not guaranteed |
384 | * to be in a keyslot. There can be more keys than keyslots. |
385 | */ |
386 | err = 0; |
387 | goto out; |
388 | } |
389 | |
390 | if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) { |
391 | /* BUG: key is still in use by I/O */ |
392 | err = -EBUSY; |
393 | goto out_remove; |
394 | } |
395 | err = profile->ll_ops.keyslot_evict(profile, key, |
396 | blk_crypto_keyslot_index(slot)); |
397 | out_remove: |
398 | /* |
399 | * Callers free the key even on error, so unlink the key from the hash |
400 | * table and clear slot->key even on error. |
401 | */ |
402 | hlist_del(n: &slot->hash_node); |
403 | slot->key = NULL; |
404 | out: |
405 | blk_crypto_hw_exit(profile); |
406 | return err; |
407 | } |
408 | |
409 | /** |
410 | * blk_crypto_reprogram_all_keys() - Re-program all keyslots. |
411 | * @profile: The crypto profile |
412 | * |
413 | * Re-program all keyslots that are supposed to have a key programmed. This is |
414 | * intended only for use by drivers for hardware that loses its keys on reset. |
415 | * |
416 | * Context: Process context. Takes and releases profile->lock. |
417 | */ |
418 | void blk_crypto_reprogram_all_keys(struct blk_crypto_profile *profile) |
419 | { |
420 | unsigned int slot; |
421 | |
422 | if (profile->num_slots == 0) |
423 | return; |
424 | |
425 | /* This is for device initialization, so don't resume the device */ |
426 | down_write(sem: &profile->lock); |
427 | for (slot = 0; slot < profile->num_slots; slot++) { |
428 | const struct blk_crypto_key *key = profile->slots[slot].key; |
429 | int err; |
430 | |
431 | if (!key) |
432 | continue; |
433 | |
434 | err = profile->ll_ops.keyslot_program(profile, key, slot); |
435 | WARN_ON(err); |
436 | } |
437 | up_write(sem: &profile->lock); |
438 | } |
439 | EXPORT_SYMBOL_GPL(blk_crypto_reprogram_all_keys); |
440 | |
441 | void blk_crypto_profile_destroy(struct blk_crypto_profile *profile) |
442 | { |
443 | if (!profile) |
444 | return; |
445 | lockdep_unregister_key(key: &profile->lockdep_key); |
446 | kvfree(addr: profile->slot_hashtable); |
447 | kvfree_sensitive(addr: profile->slots, |
448 | len: sizeof(profile->slots[0]) * profile->num_slots); |
449 | memzero_explicit(s: profile, count: sizeof(*profile)); |
450 | } |
451 | EXPORT_SYMBOL_GPL(blk_crypto_profile_destroy); |
452 | |
453 | bool blk_crypto_register(struct blk_crypto_profile *profile, |
454 | struct request_queue *q) |
455 | { |
456 | if (blk_integrity_queue_supports_integrity(q)) { |
457 | pr_warn("Integrity and hardware inline encryption are not supported together. Disabling hardware inline encryption.\n" ); |
458 | return false; |
459 | } |
460 | q->crypto_profile = profile; |
461 | return true; |
462 | } |
463 | EXPORT_SYMBOL_GPL(blk_crypto_register); |
464 | |
465 | /** |
466 | * blk_crypto_intersect_capabilities() - restrict supported crypto capabilities |
467 | * by child device |
468 | * @parent: the crypto profile for the parent device |
469 | * @child: the crypto profile for the child device, or NULL |
470 | * |
471 | * This clears all crypto capabilities in @parent that aren't set in @child. If |
472 | * @child is NULL, then this clears all parent capabilities. |
473 | * |
474 | * Only use this when setting up the crypto profile for a layered device, before |
475 | * it's been exposed yet. |
476 | */ |
477 | void blk_crypto_intersect_capabilities(struct blk_crypto_profile *parent, |
478 | const struct blk_crypto_profile *child) |
479 | { |
480 | if (child) { |
481 | unsigned int i; |
482 | |
483 | parent->max_dun_bytes_supported = |
484 | min(parent->max_dun_bytes_supported, |
485 | child->max_dun_bytes_supported); |
486 | for (i = 0; i < ARRAY_SIZE(child->modes_supported); i++) |
487 | parent->modes_supported[i] &= child->modes_supported[i]; |
488 | } else { |
489 | parent->max_dun_bytes_supported = 0; |
490 | memset(parent->modes_supported, 0, |
491 | sizeof(parent->modes_supported)); |
492 | } |
493 | } |
494 | EXPORT_SYMBOL_GPL(blk_crypto_intersect_capabilities); |
495 | |
496 | /** |
497 | * blk_crypto_has_capabilities() - Check whether @target supports at least all |
498 | * the crypto capabilities that @reference does. |
499 | * @target: the target profile |
500 | * @reference: the reference profile |
501 | * |
502 | * Return: %true if @target supports all the crypto capabilities of @reference. |
503 | */ |
504 | bool blk_crypto_has_capabilities(const struct blk_crypto_profile *target, |
505 | const struct blk_crypto_profile *reference) |
506 | { |
507 | int i; |
508 | |
509 | if (!reference) |
510 | return true; |
511 | |
512 | if (!target) |
513 | return false; |
514 | |
515 | for (i = 0; i < ARRAY_SIZE(target->modes_supported); i++) { |
516 | if (reference->modes_supported[i] & ~target->modes_supported[i]) |
517 | return false; |
518 | } |
519 | |
520 | if (reference->max_dun_bytes_supported > |
521 | target->max_dun_bytes_supported) |
522 | return false; |
523 | |
524 | return true; |
525 | } |
526 | EXPORT_SYMBOL_GPL(blk_crypto_has_capabilities); |
527 | |
528 | /** |
529 | * blk_crypto_update_capabilities() - Update the capabilities of a crypto |
530 | * profile to match those of another crypto |
531 | * profile. |
532 | * @dst: The crypto profile whose capabilities to update. |
533 | * @src: The crypto profile whose capabilities this function will update @dst's |
534 | * capabilities to. |
535 | * |
536 | * Blk-crypto requires that crypto capabilities that were |
537 | * advertised when a bio was created continue to be supported by the |
538 | * device until that bio is ended. This is turn means that a device cannot |
539 | * shrink its advertised crypto capabilities without any explicit |
540 | * synchronization with upper layers. So if there's no such explicit |
541 | * synchronization, @src must support all the crypto capabilities that |
542 | * @dst does (i.e. we need blk_crypto_has_capabilities(@src, @dst)). |
543 | * |
544 | * Note also that as long as the crypto capabilities are being expanded, the |
545 | * order of updates becoming visible is not important because it's alright |
546 | * for blk-crypto to see stale values - they only cause blk-crypto to |
547 | * believe that a crypto capability isn't supported when it actually is (which |
548 | * might result in blk-crypto-fallback being used if available, or the bio being |
549 | * failed). |
550 | */ |
551 | void blk_crypto_update_capabilities(struct blk_crypto_profile *dst, |
552 | const struct blk_crypto_profile *src) |
553 | { |
554 | memcpy(dst->modes_supported, src->modes_supported, |
555 | sizeof(dst->modes_supported)); |
556 | |
557 | dst->max_dun_bytes_supported = src->max_dun_bytes_supported; |
558 | } |
559 | EXPORT_SYMBOL_GPL(blk_crypto_update_capabilities); |
560 | |