1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ |
3 | |
4 | #include <linux/kernel.h> |
5 | #include <linux/module.h> |
6 | #include <crypto/algapi.h> |
7 | #include <crypto/internal/skcipher.h> |
8 | #include <crypto/internal/des.h> |
9 | #include <crypto/xts.h> |
10 | #include <crypto/sm4.h> |
11 | #include <crypto/scatterwalk.h> |
12 | |
13 | #include "cc_driver.h" |
14 | #include "cc_lli_defs.h" |
15 | #include "cc_buffer_mgr.h" |
16 | #include "cc_cipher.h" |
17 | #include "cc_request_mgr.h" |
18 | |
19 | #define MAX_SKCIPHER_SEQ_LEN 6 |
20 | |
21 | #define template_skcipher template_u.skcipher |
22 | |
23 | struct cc_user_key_info { |
24 | u8 *key; |
25 | dma_addr_t key_dma_addr; |
26 | }; |
27 | |
28 | struct cc_hw_key_info { |
29 | enum cc_hw_crypto_key key1_slot; |
30 | enum cc_hw_crypto_key key2_slot; |
31 | }; |
32 | |
33 | struct cc_cpp_key_info { |
34 | u8 slot; |
35 | enum cc_cpp_alg alg; |
36 | }; |
37 | |
38 | enum cc_key_type { |
39 | CC_UNPROTECTED_KEY, /* User key */ |
40 | CC_HW_PROTECTED_KEY, /* HW (FDE) key */ |
41 | CC_POLICY_PROTECTED_KEY, /* CPP key */ |
42 | CC_INVALID_PROTECTED_KEY /* Invalid key */ |
43 | }; |
44 | |
45 | struct cc_cipher_ctx { |
46 | struct cc_drvdata *drvdata; |
47 | int keylen; |
48 | int cipher_mode; |
49 | int flow_mode; |
50 | unsigned int flags; |
51 | enum cc_key_type key_type; |
52 | struct cc_user_key_info user; |
53 | union { |
54 | struct cc_hw_key_info hw; |
55 | struct cc_cpp_key_info cpp; |
56 | }; |
57 | struct crypto_shash *shash_tfm; |
58 | struct crypto_skcipher *fallback_tfm; |
59 | bool fallback_on; |
60 | }; |
61 | |
62 | static void cc_cipher_complete(struct device *dev, void *cc_req, int err); |
63 | |
64 | static inline enum cc_key_type cc_key_type(struct crypto_tfm *tfm) |
65 | { |
66 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
67 | |
68 | return ctx_p->key_type; |
69 | } |
70 | |
71 | static int validate_keys_sizes(struct cc_cipher_ctx *ctx_p, u32 size) |
72 | { |
73 | switch (ctx_p->flow_mode) { |
74 | case S_DIN_to_AES: |
75 | switch (size) { |
76 | case CC_AES_128_BIT_KEY_SIZE: |
77 | case CC_AES_192_BIT_KEY_SIZE: |
78 | if (ctx_p->cipher_mode != DRV_CIPHER_XTS) |
79 | return 0; |
80 | break; |
81 | case CC_AES_256_BIT_KEY_SIZE: |
82 | return 0; |
83 | case (CC_AES_192_BIT_KEY_SIZE * 2): |
84 | case (CC_AES_256_BIT_KEY_SIZE * 2): |
85 | if (ctx_p->cipher_mode == DRV_CIPHER_XTS || |
86 | ctx_p->cipher_mode == DRV_CIPHER_ESSIV) |
87 | return 0; |
88 | break; |
89 | default: |
90 | break; |
91 | } |
92 | break; |
93 | case S_DIN_to_DES: |
94 | if (size == DES3_EDE_KEY_SIZE || size == DES_KEY_SIZE) |
95 | return 0; |
96 | break; |
97 | case S_DIN_to_SM4: |
98 | if (size == SM4_KEY_SIZE) |
99 | return 0; |
100 | break; |
101 | default: |
102 | break; |
103 | } |
104 | return -EINVAL; |
105 | } |
106 | |
107 | static int validate_data_size(struct cc_cipher_ctx *ctx_p, |
108 | unsigned int size) |
109 | { |
110 | switch (ctx_p->flow_mode) { |
111 | case S_DIN_to_AES: |
112 | switch (ctx_p->cipher_mode) { |
113 | case DRV_CIPHER_XTS: |
114 | case DRV_CIPHER_CBC_CTS: |
115 | if (size >= AES_BLOCK_SIZE) |
116 | return 0; |
117 | break; |
118 | case DRV_CIPHER_OFB: |
119 | case DRV_CIPHER_CTR: |
120 | return 0; |
121 | case DRV_CIPHER_ECB: |
122 | case DRV_CIPHER_CBC: |
123 | case DRV_CIPHER_ESSIV: |
124 | if (IS_ALIGNED(size, AES_BLOCK_SIZE)) |
125 | return 0; |
126 | break; |
127 | default: |
128 | break; |
129 | } |
130 | break; |
131 | case S_DIN_to_DES: |
132 | if (IS_ALIGNED(size, DES_BLOCK_SIZE)) |
133 | return 0; |
134 | break; |
135 | case S_DIN_to_SM4: |
136 | switch (ctx_p->cipher_mode) { |
137 | case DRV_CIPHER_CTR: |
138 | return 0; |
139 | case DRV_CIPHER_ECB: |
140 | case DRV_CIPHER_CBC: |
141 | if (IS_ALIGNED(size, SM4_BLOCK_SIZE)) |
142 | return 0; |
143 | break; |
144 | default: |
145 | break; |
146 | } |
147 | break; |
148 | default: |
149 | break; |
150 | } |
151 | return -EINVAL; |
152 | } |
153 | |
154 | static int cc_cipher_init(struct crypto_tfm *tfm) |
155 | { |
156 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
157 | struct cc_crypto_alg *cc_alg = |
158 | container_of(tfm->__crt_alg, struct cc_crypto_alg, |
159 | skcipher_alg.base); |
160 | struct device *dev = drvdata_to_dev(drvdata: cc_alg->drvdata); |
161 | unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; |
162 | unsigned int fallback_req_size = 0; |
163 | |
164 | dev_dbg(dev, "Initializing context @%p for %s\n" , ctx_p, |
165 | crypto_tfm_alg_name(tfm)); |
166 | |
167 | ctx_p->cipher_mode = cc_alg->cipher_mode; |
168 | ctx_p->flow_mode = cc_alg->flow_mode; |
169 | ctx_p->drvdata = cc_alg->drvdata; |
170 | |
171 | if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { |
172 | const char *name = crypto_tfm_alg_name(tfm); |
173 | |
174 | /* Alloc hash tfm for essiv */ |
175 | ctx_p->shash_tfm = crypto_alloc_shash(alg_name: "sha256" , type: 0, mask: 0); |
176 | if (IS_ERR(ptr: ctx_p->shash_tfm)) { |
177 | dev_err(dev, "Error allocating hash tfm for ESSIV.\n" ); |
178 | return PTR_ERR(ptr: ctx_p->shash_tfm); |
179 | } |
180 | max_key_buf_size <<= 1; |
181 | |
182 | /* Alloc fallabck tfm or essiv when key size != 256 bit */ |
183 | ctx_p->fallback_tfm = |
184 | crypto_alloc_skcipher(alg_name: name, type: 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); |
185 | |
186 | if (IS_ERR(ptr: ctx_p->fallback_tfm)) { |
187 | /* Note we're still allowing registration with no fallback since it's |
188 | * better to have most modes supported than none at all. |
189 | */ |
190 | dev_warn(dev, "Error allocating fallback algo %s. Some modes may be available.\n" , |
191 | name); |
192 | ctx_p->fallback_tfm = NULL; |
193 | } else { |
194 | fallback_req_size = crypto_skcipher_reqsize(tfm: ctx_p->fallback_tfm); |
195 | } |
196 | } |
197 | |
198 | crypto_skcipher_set_reqsize(skcipher: __crypto_skcipher_cast(tfm), |
199 | reqsize: sizeof(struct cipher_req_ctx) + fallback_req_size); |
200 | |
201 | /* Allocate key buffer, cache line aligned */ |
202 | ctx_p->user.key = kzalloc(size: max_key_buf_size, GFP_KERNEL); |
203 | if (!ctx_p->user.key) |
204 | goto free_fallback; |
205 | |
206 | dev_dbg(dev, "Allocated key buffer in context. key=@%p\n" , |
207 | ctx_p->user.key); |
208 | |
209 | /* Map key buffer */ |
210 | ctx_p->user.key_dma_addr = dma_map_single(dev, ctx_p->user.key, |
211 | max_key_buf_size, |
212 | DMA_TO_DEVICE); |
213 | if (dma_mapping_error(dev, dma_addr: ctx_p->user.key_dma_addr)) { |
214 | dev_err(dev, "Mapping Key %u B at va=%pK for DMA failed\n" , |
215 | max_key_buf_size, ctx_p->user.key); |
216 | goto free_key; |
217 | } |
218 | dev_dbg(dev, "Mapped key %u B at va=%pK to dma=%pad\n" , |
219 | max_key_buf_size, ctx_p->user.key, &ctx_p->user.key_dma_addr); |
220 | |
221 | return 0; |
222 | |
223 | free_key: |
224 | kfree(objp: ctx_p->user.key); |
225 | free_fallback: |
226 | crypto_free_skcipher(tfm: ctx_p->fallback_tfm); |
227 | crypto_free_shash(tfm: ctx_p->shash_tfm); |
228 | |
229 | return -ENOMEM; |
230 | } |
231 | |
232 | static void cc_cipher_exit(struct crypto_tfm *tfm) |
233 | { |
234 | struct crypto_alg *alg = tfm->__crt_alg; |
235 | struct cc_crypto_alg *cc_alg = |
236 | container_of(alg, struct cc_crypto_alg, |
237 | skcipher_alg.base); |
238 | unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; |
239 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
240 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
241 | |
242 | dev_dbg(dev, "Clearing context @%p for %s\n" , |
243 | crypto_tfm_ctx(tfm), crypto_tfm_alg_name(tfm)); |
244 | |
245 | if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { |
246 | /* Free hash tfm for essiv */ |
247 | crypto_free_shash(tfm: ctx_p->shash_tfm); |
248 | ctx_p->shash_tfm = NULL; |
249 | crypto_free_skcipher(tfm: ctx_p->fallback_tfm); |
250 | ctx_p->fallback_tfm = NULL; |
251 | } |
252 | |
253 | /* Unmap key buffer */ |
254 | dma_unmap_single(dev, ctx_p->user.key_dma_addr, max_key_buf_size, |
255 | DMA_TO_DEVICE); |
256 | dev_dbg(dev, "Unmapped key buffer key_dma_addr=%pad\n" , |
257 | &ctx_p->user.key_dma_addr); |
258 | |
259 | /* Free key buffer in context */ |
260 | dev_dbg(dev, "Free key buffer in context. key=@%p\n" , ctx_p->user.key); |
261 | kfree_sensitive(objp: ctx_p->user.key); |
262 | } |
263 | |
264 | struct tdes_keys { |
265 | u8 key1[DES_KEY_SIZE]; |
266 | u8 key2[DES_KEY_SIZE]; |
267 | u8 key3[DES_KEY_SIZE]; |
268 | }; |
269 | |
270 | static enum cc_hw_crypto_key cc_slot_to_hw_key(u8 slot_num) |
271 | { |
272 | switch (slot_num) { |
273 | case 0: |
274 | return KFDE0_KEY; |
275 | case 1: |
276 | return KFDE1_KEY; |
277 | case 2: |
278 | return KFDE2_KEY; |
279 | case 3: |
280 | return KFDE3_KEY; |
281 | } |
282 | return END_OF_KEYS; |
283 | } |
284 | |
285 | static u8 cc_slot_to_cpp_key(u8 slot_num) |
286 | { |
287 | return (slot_num - CC_FIRST_CPP_KEY_SLOT); |
288 | } |
289 | |
290 | static inline enum cc_key_type cc_slot_to_key_type(u8 slot_num) |
291 | { |
292 | if (slot_num >= CC_FIRST_HW_KEY_SLOT && slot_num <= CC_LAST_HW_KEY_SLOT) |
293 | return CC_HW_PROTECTED_KEY; |
294 | else if (slot_num >= CC_FIRST_CPP_KEY_SLOT && |
295 | slot_num <= CC_LAST_CPP_KEY_SLOT) |
296 | return CC_POLICY_PROTECTED_KEY; |
297 | else |
298 | return CC_INVALID_PROTECTED_KEY; |
299 | } |
300 | |
301 | static int cc_cipher_sethkey(struct crypto_skcipher *sktfm, const u8 *key, |
302 | unsigned int keylen) |
303 | { |
304 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: sktfm); |
305 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
306 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
307 | struct cc_hkey_info hki; |
308 | |
309 | dev_dbg(dev, "Setting HW key in context @%p for %s. keylen=%u\n" , |
310 | ctx_p, crypto_tfm_alg_name(tfm), keylen); |
311 | dump_byte_array(name: "key" , the_array: key, size: keylen); |
312 | |
313 | /* STAT_PHASE_0: Init and sanity checks */ |
314 | |
315 | /* This check the size of the protected key token */ |
316 | if (keylen != sizeof(hki)) { |
317 | dev_err(dev, "Unsupported protected key size %d.\n" , keylen); |
318 | return -EINVAL; |
319 | } |
320 | |
321 | memcpy(&hki, key, keylen); |
322 | |
323 | /* The real key len for crypto op is the size of the HW key |
324 | * referenced by the HW key slot, not the hardware key token |
325 | */ |
326 | keylen = hki.keylen; |
327 | |
328 | if (validate_keys_sizes(ctx_p, size: keylen)) { |
329 | dev_dbg(dev, "Unsupported key size %d.\n" , keylen); |
330 | return -EINVAL; |
331 | } |
332 | |
333 | ctx_p->keylen = keylen; |
334 | ctx_p->fallback_on = false; |
335 | |
336 | switch (cc_slot_to_key_type(slot_num: hki.hw_key1)) { |
337 | case CC_HW_PROTECTED_KEY: |
338 | if (ctx_p->flow_mode == S_DIN_to_SM4) { |
339 | dev_err(dev, "Only AES HW protected keys are supported\n" ); |
340 | return -EINVAL; |
341 | } |
342 | |
343 | ctx_p->hw.key1_slot = cc_slot_to_hw_key(slot_num: hki.hw_key1); |
344 | if (ctx_p->hw.key1_slot == END_OF_KEYS) { |
345 | dev_err(dev, "Unsupported hw key1 number (%d)\n" , |
346 | hki.hw_key1); |
347 | return -EINVAL; |
348 | } |
349 | |
350 | if (ctx_p->cipher_mode == DRV_CIPHER_XTS || |
351 | ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { |
352 | if (hki.hw_key1 == hki.hw_key2) { |
353 | dev_err(dev, "Illegal hw key numbers (%d,%d)\n" , |
354 | hki.hw_key1, hki.hw_key2); |
355 | return -EINVAL; |
356 | } |
357 | |
358 | ctx_p->hw.key2_slot = cc_slot_to_hw_key(slot_num: hki.hw_key2); |
359 | if (ctx_p->hw.key2_slot == END_OF_KEYS) { |
360 | dev_err(dev, "Unsupported hw key2 number (%d)\n" , |
361 | hki.hw_key2); |
362 | return -EINVAL; |
363 | } |
364 | } |
365 | |
366 | ctx_p->key_type = CC_HW_PROTECTED_KEY; |
367 | dev_dbg(dev, "HW protected key %d/%d set\n." , |
368 | ctx_p->hw.key1_slot, ctx_p->hw.key2_slot); |
369 | break; |
370 | |
371 | case CC_POLICY_PROTECTED_KEY: |
372 | if (ctx_p->drvdata->hw_rev < CC_HW_REV_713) { |
373 | dev_err(dev, "CPP keys not supported in this hardware revision.\n" ); |
374 | return -EINVAL; |
375 | } |
376 | |
377 | if (ctx_p->cipher_mode != DRV_CIPHER_CBC && |
378 | ctx_p->cipher_mode != DRV_CIPHER_CTR) { |
379 | dev_err(dev, "CPP keys only supported in CBC or CTR modes.\n" ); |
380 | return -EINVAL; |
381 | } |
382 | |
383 | ctx_p->cpp.slot = cc_slot_to_cpp_key(slot_num: hki.hw_key1); |
384 | if (ctx_p->flow_mode == S_DIN_to_AES) |
385 | ctx_p->cpp.alg = CC_CPP_AES; |
386 | else /* Must be SM4 since due to sethkey registration */ |
387 | ctx_p->cpp.alg = CC_CPP_SM4; |
388 | ctx_p->key_type = CC_POLICY_PROTECTED_KEY; |
389 | dev_dbg(dev, "policy protected key alg: %d slot: %d.\n" , |
390 | ctx_p->cpp.alg, ctx_p->cpp.slot); |
391 | break; |
392 | |
393 | default: |
394 | dev_err(dev, "Unsupported protected key (%d)\n" , hki.hw_key1); |
395 | return -EINVAL; |
396 | } |
397 | |
398 | return 0; |
399 | } |
400 | |
401 | static int cc_cipher_setkey(struct crypto_skcipher *sktfm, const u8 *key, |
402 | unsigned int keylen) |
403 | { |
404 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: sktfm); |
405 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
406 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
407 | struct cc_crypto_alg *cc_alg = |
408 | container_of(tfm->__crt_alg, struct cc_crypto_alg, |
409 | skcipher_alg.base); |
410 | unsigned int max_key_buf_size = cc_alg->skcipher_alg.max_keysize; |
411 | |
412 | dev_dbg(dev, "Setting key in context @%p for %s. keylen=%u\n" , |
413 | ctx_p, crypto_tfm_alg_name(tfm), keylen); |
414 | dump_byte_array(name: "key" , the_array: key, size: keylen); |
415 | |
416 | /* STAT_PHASE_0: Init and sanity checks */ |
417 | |
418 | if (validate_keys_sizes(ctx_p, size: keylen)) { |
419 | dev_dbg(dev, "Invalid key size %d.\n" , keylen); |
420 | return -EINVAL; |
421 | } |
422 | |
423 | if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { |
424 | |
425 | /* We only support 256 bit ESSIV-CBC-AES keys */ |
426 | if (keylen != AES_KEYSIZE_256) { |
427 | unsigned int flags = crypto_tfm_get_flags(tfm) & CRYPTO_TFM_REQ_MASK; |
428 | |
429 | if (likely(ctx_p->fallback_tfm)) { |
430 | ctx_p->fallback_on = true; |
431 | crypto_skcipher_clear_flags(tfm: ctx_p->fallback_tfm, |
432 | CRYPTO_TFM_REQ_MASK); |
433 | crypto_skcipher_clear_flags(tfm: ctx_p->fallback_tfm, flags); |
434 | return crypto_skcipher_setkey(tfm: ctx_p->fallback_tfm, key, keylen); |
435 | } |
436 | |
437 | dev_dbg(dev, "Unsupported key size %d and no fallback.\n" , keylen); |
438 | return -EINVAL; |
439 | } |
440 | |
441 | /* Internal ESSIV key buffer is double sized */ |
442 | max_key_buf_size <<= 1; |
443 | } |
444 | |
445 | ctx_p->fallback_on = false; |
446 | ctx_p->key_type = CC_UNPROTECTED_KEY; |
447 | |
448 | /* |
449 | * Verify DES weak keys |
450 | * Note that we're dropping the expanded key since the |
451 | * HW does the expansion on its own. |
452 | */ |
453 | if (ctx_p->flow_mode == S_DIN_to_DES) { |
454 | if ((keylen == DES3_EDE_KEY_SIZE && |
455 | verify_skcipher_des3_key(tfm: sktfm, key)) || |
456 | verify_skcipher_des_key(tfm: sktfm, key)) { |
457 | dev_dbg(dev, "weak DES key" ); |
458 | return -EINVAL; |
459 | } |
460 | } |
461 | |
462 | if (ctx_p->cipher_mode == DRV_CIPHER_XTS && |
463 | xts_verify_key(tfm: sktfm, key, keylen)) { |
464 | dev_dbg(dev, "weak XTS key" ); |
465 | return -EINVAL; |
466 | } |
467 | |
468 | /* STAT_PHASE_1: Copy key to ctx */ |
469 | dma_sync_single_for_cpu(dev, addr: ctx_p->user.key_dma_addr, |
470 | size: max_key_buf_size, dir: DMA_TO_DEVICE); |
471 | |
472 | memcpy(ctx_p->user.key, key, keylen); |
473 | |
474 | if (ctx_p->cipher_mode == DRV_CIPHER_ESSIV) { |
475 | /* sha256 for key2 - use sw implementation */ |
476 | int err; |
477 | |
478 | err = crypto_shash_tfm_digest(tfm: ctx_p->shash_tfm, |
479 | data: ctx_p->user.key, len: keylen, |
480 | out: ctx_p->user.key + keylen); |
481 | if (err) { |
482 | dev_err(dev, "Failed to hash ESSIV key.\n" ); |
483 | return err; |
484 | } |
485 | |
486 | keylen <<= 1; |
487 | } |
488 | dma_sync_single_for_device(dev, addr: ctx_p->user.key_dma_addr, |
489 | size: max_key_buf_size, dir: DMA_TO_DEVICE); |
490 | ctx_p->keylen = keylen; |
491 | |
492 | dev_dbg(dev, "return safely" ); |
493 | return 0; |
494 | } |
495 | |
496 | static int cc_out_setup_mode(struct cc_cipher_ctx *ctx_p) |
497 | { |
498 | switch (ctx_p->flow_mode) { |
499 | case S_DIN_to_AES: |
500 | return S_AES_to_DOUT; |
501 | case S_DIN_to_DES: |
502 | return S_DES_to_DOUT; |
503 | case S_DIN_to_SM4: |
504 | return S_SM4_to_DOUT; |
505 | default: |
506 | return ctx_p->flow_mode; |
507 | } |
508 | } |
509 | |
510 | static void cc_setup_readiv_desc(struct crypto_tfm *tfm, |
511 | struct cipher_req_ctx *req_ctx, |
512 | unsigned int ivsize, struct cc_hw_desc desc[], |
513 | unsigned int *seq_size) |
514 | { |
515 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
516 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
517 | int cipher_mode = ctx_p->cipher_mode; |
518 | int flow_mode = cc_out_setup_mode(ctx_p); |
519 | int direction = req_ctx->gen_ctx.op_type; |
520 | dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; |
521 | |
522 | if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) |
523 | return; |
524 | |
525 | switch (cipher_mode) { |
526 | case DRV_CIPHER_ECB: |
527 | break; |
528 | case DRV_CIPHER_CBC: |
529 | case DRV_CIPHER_CBC_CTS: |
530 | case DRV_CIPHER_CTR: |
531 | case DRV_CIPHER_OFB: |
532 | /* Read next IV */ |
533 | hw_desc_init(pdesc: &desc[*seq_size]); |
534 | set_dout_dlli(pdesc: &desc[*seq_size], addr: iv_dma_addr, size: ivsize, NS_BIT, last_ind: 1); |
535 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
536 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
537 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
538 | if (cipher_mode == DRV_CIPHER_CTR || |
539 | cipher_mode == DRV_CIPHER_OFB) { |
540 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_WRITE_STATE1); |
541 | } else { |
542 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_WRITE_STATE0); |
543 | } |
544 | set_queue_last_ind(drvdata: ctx_p->drvdata, pdesc: &desc[*seq_size]); |
545 | (*seq_size)++; |
546 | break; |
547 | case DRV_CIPHER_XTS: |
548 | case DRV_CIPHER_ESSIV: |
549 | /* IV */ |
550 | hw_desc_init(pdesc: &desc[*seq_size]); |
551 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_WRITE_STATE1); |
552 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
553 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
554 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
555 | set_dout_dlli(pdesc: &desc[*seq_size], addr: iv_dma_addr, CC_AES_BLOCK_SIZE, |
556 | NS_BIT, last_ind: 1); |
557 | set_queue_last_ind(drvdata: ctx_p->drvdata, pdesc: &desc[*seq_size]); |
558 | (*seq_size)++; |
559 | break; |
560 | default: |
561 | dev_err(dev, "Unsupported cipher mode (%d)\n" , cipher_mode); |
562 | } |
563 | } |
564 | |
565 | |
566 | static void cc_setup_state_desc(struct crypto_tfm *tfm, |
567 | struct cipher_req_ctx *req_ctx, |
568 | unsigned int ivsize, unsigned int nbytes, |
569 | struct cc_hw_desc desc[], |
570 | unsigned int *seq_size) |
571 | { |
572 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
573 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
574 | int cipher_mode = ctx_p->cipher_mode; |
575 | int flow_mode = ctx_p->flow_mode; |
576 | int direction = req_ctx->gen_ctx.op_type; |
577 | dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; |
578 | |
579 | switch (cipher_mode) { |
580 | case DRV_CIPHER_ECB: |
581 | break; |
582 | case DRV_CIPHER_CBC: |
583 | case DRV_CIPHER_CBC_CTS: |
584 | case DRV_CIPHER_CTR: |
585 | case DRV_CIPHER_OFB: |
586 | /* Load IV */ |
587 | hw_desc_init(pdesc: &desc[*seq_size]); |
588 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, addr: iv_dma_addr, size: ivsize, |
589 | NS_BIT); |
590 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
591 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
592 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
593 | if (cipher_mode == DRV_CIPHER_CTR || |
594 | cipher_mode == DRV_CIPHER_OFB) { |
595 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_STATE1); |
596 | } else { |
597 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_STATE0); |
598 | } |
599 | (*seq_size)++; |
600 | break; |
601 | case DRV_CIPHER_XTS: |
602 | case DRV_CIPHER_ESSIV: |
603 | break; |
604 | default: |
605 | dev_err(dev, "Unsupported cipher mode (%d)\n" , cipher_mode); |
606 | } |
607 | } |
608 | |
609 | |
610 | static void cc_setup_xex_state_desc(struct crypto_tfm *tfm, |
611 | struct cipher_req_ctx *req_ctx, |
612 | unsigned int ivsize, unsigned int nbytes, |
613 | struct cc_hw_desc desc[], |
614 | unsigned int *seq_size) |
615 | { |
616 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
617 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
618 | int cipher_mode = ctx_p->cipher_mode; |
619 | int flow_mode = ctx_p->flow_mode; |
620 | int direction = req_ctx->gen_ctx.op_type; |
621 | dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; |
622 | unsigned int key_len = (ctx_p->keylen / 2); |
623 | dma_addr_t iv_dma_addr = req_ctx->gen_ctx.iv_dma_addr; |
624 | unsigned int key_offset = key_len; |
625 | |
626 | switch (cipher_mode) { |
627 | case DRV_CIPHER_ECB: |
628 | break; |
629 | case DRV_CIPHER_CBC: |
630 | case DRV_CIPHER_CBC_CTS: |
631 | case DRV_CIPHER_CTR: |
632 | case DRV_CIPHER_OFB: |
633 | break; |
634 | case DRV_CIPHER_XTS: |
635 | case DRV_CIPHER_ESSIV: |
636 | |
637 | if (cipher_mode == DRV_CIPHER_ESSIV) |
638 | key_len = SHA256_DIGEST_SIZE; |
639 | |
640 | /* load XEX key */ |
641 | hw_desc_init(pdesc: &desc[*seq_size]); |
642 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
643 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
644 | if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { |
645 | set_hw_crypto_key(pdesc: &desc[*seq_size], |
646 | hw_key: ctx_p->hw.key2_slot); |
647 | } else { |
648 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, |
649 | addr: (key_dma_addr + key_offset), |
650 | size: key_len, NS_BIT); |
651 | } |
652 | set_xex_data_unit_size(pdesc: &desc[*seq_size], size: nbytes); |
653 | set_flow_mode(pdesc: &desc[*seq_size], mode: S_DIN_to_AES2); |
654 | set_key_size_aes(pdesc: &desc[*seq_size], size: key_len); |
655 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_XEX_KEY); |
656 | (*seq_size)++; |
657 | |
658 | /* Load IV */ |
659 | hw_desc_init(pdesc: &desc[*seq_size]); |
660 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_STATE1); |
661 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
662 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
663 | set_key_size_aes(pdesc: &desc[*seq_size], size: key_len); |
664 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
665 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, addr: iv_dma_addr, |
666 | CC_AES_BLOCK_SIZE, NS_BIT); |
667 | (*seq_size)++; |
668 | break; |
669 | default: |
670 | dev_err(dev, "Unsupported cipher mode (%d)\n" , cipher_mode); |
671 | } |
672 | } |
673 | |
674 | static int cc_out_flow_mode(struct cc_cipher_ctx *ctx_p) |
675 | { |
676 | switch (ctx_p->flow_mode) { |
677 | case S_DIN_to_AES: |
678 | return DIN_AES_DOUT; |
679 | case S_DIN_to_DES: |
680 | return DIN_DES_DOUT; |
681 | case S_DIN_to_SM4: |
682 | return DIN_SM4_DOUT; |
683 | default: |
684 | return ctx_p->flow_mode; |
685 | } |
686 | } |
687 | |
688 | static void cc_setup_key_desc(struct crypto_tfm *tfm, |
689 | struct cipher_req_ctx *req_ctx, |
690 | unsigned int nbytes, struct cc_hw_desc desc[], |
691 | unsigned int *seq_size) |
692 | { |
693 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
694 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
695 | int cipher_mode = ctx_p->cipher_mode; |
696 | int flow_mode = ctx_p->flow_mode; |
697 | int direction = req_ctx->gen_ctx.op_type; |
698 | dma_addr_t key_dma_addr = ctx_p->user.key_dma_addr; |
699 | unsigned int key_len = ctx_p->keylen; |
700 | unsigned int din_size; |
701 | |
702 | switch (cipher_mode) { |
703 | case DRV_CIPHER_CBC: |
704 | case DRV_CIPHER_CBC_CTS: |
705 | case DRV_CIPHER_CTR: |
706 | case DRV_CIPHER_OFB: |
707 | case DRV_CIPHER_ECB: |
708 | /* Load key */ |
709 | hw_desc_init(pdesc: &desc[*seq_size]); |
710 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
711 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
712 | |
713 | if (cc_key_type(tfm) == CC_POLICY_PROTECTED_KEY) { |
714 | /* We use the AES key size coding for all CPP algs */ |
715 | set_key_size_aes(pdesc: &desc[*seq_size], size: key_len); |
716 | set_cpp_crypto_key(pdesc: &desc[*seq_size], slot: ctx_p->cpp.slot); |
717 | flow_mode = cc_out_flow_mode(ctx_p); |
718 | } else { |
719 | if (flow_mode == S_DIN_to_AES) { |
720 | if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { |
721 | set_hw_crypto_key(pdesc: &desc[*seq_size], |
722 | hw_key: ctx_p->hw.key1_slot); |
723 | } else { |
724 | /* CC_POLICY_UNPROTECTED_KEY |
725 | * Invalid keys are filtered out in |
726 | * sethkey() |
727 | */ |
728 | din_size = (key_len == 24) ? |
729 | AES_MAX_KEY_SIZE : key_len; |
730 | |
731 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, |
732 | addr: key_dma_addr, size: din_size, |
733 | NS_BIT); |
734 | } |
735 | set_key_size_aes(pdesc: &desc[*seq_size], size: key_len); |
736 | } else { |
737 | /*des*/ |
738 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, |
739 | addr: key_dma_addr, size: key_len, NS_BIT); |
740 | set_key_size_des(pdesc: &desc[*seq_size], size: key_len); |
741 | } |
742 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_KEY0); |
743 | } |
744 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
745 | (*seq_size)++; |
746 | break; |
747 | case DRV_CIPHER_XTS: |
748 | case DRV_CIPHER_ESSIV: |
749 | /* Load AES key */ |
750 | hw_desc_init(pdesc: &desc[*seq_size]); |
751 | set_cipher_mode(pdesc: &desc[*seq_size], mode: cipher_mode); |
752 | set_cipher_config0(pdesc: &desc[*seq_size], mode: direction); |
753 | if (cc_key_type(tfm) == CC_HW_PROTECTED_KEY) { |
754 | set_hw_crypto_key(pdesc: &desc[*seq_size], |
755 | hw_key: ctx_p->hw.key1_slot); |
756 | } else { |
757 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, addr: key_dma_addr, |
758 | size: (key_len / 2), NS_BIT); |
759 | } |
760 | set_key_size_aes(pdesc: &desc[*seq_size], size: (key_len / 2)); |
761 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
762 | set_setup_mode(pdesc: &desc[*seq_size], mode: SETUP_LOAD_KEY0); |
763 | (*seq_size)++; |
764 | break; |
765 | default: |
766 | dev_err(dev, "Unsupported cipher mode (%d)\n" , cipher_mode); |
767 | } |
768 | } |
769 | |
770 | static void cc_setup_mlli_desc(struct crypto_tfm *tfm, |
771 | struct cipher_req_ctx *req_ctx, |
772 | struct scatterlist *dst, struct scatterlist *src, |
773 | unsigned int nbytes, void *areq, |
774 | struct cc_hw_desc desc[], unsigned int *seq_size) |
775 | { |
776 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
777 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
778 | |
779 | if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { |
780 | /* bypass */ |
781 | dev_dbg(dev, " bypass params addr %pad length 0x%X addr 0x%08X\n" , |
782 | &req_ctx->mlli_params.mlli_dma_addr, |
783 | req_ctx->mlli_params.mlli_len, |
784 | ctx_p->drvdata->mlli_sram_addr); |
785 | hw_desc_init(pdesc: &desc[*seq_size]); |
786 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, |
787 | addr: req_ctx->mlli_params.mlli_dma_addr, |
788 | size: req_ctx->mlli_params.mlli_len, NS_BIT); |
789 | set_dout_sram(pdesc: &desc[*seq_size], |
790 | addr: ctx_p->drvdata->mlli_sram_addr, |
791 | size: req_ctx->mlli_params.mlli_len); |
792 | set_flow_mode(pdesc: &desc[*seq_size], mode: BYPASS); |
793 | (*seq_size)++; |
794 | } |
795 | } |
796 | |
797 | static void cc_setup_flow_desc(struct crypto_tfm *tfm, |
798 | struct cipher_req_ctx *req_ctx, |
799 | struct scatterlist *dst, struct scatterlist *src, |
800 | unsigned int nbytes, struct cc_hw_desc desc[], |
801 | unsigned int *seq_size) |
802 | { |
803 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
804 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
805 | unsigned int flow_mode = cc_out_flow_mode(ctx_p); |
806 | bool last_desc = (ctx_p->key_type == CC_POLICY_PROTECTED_KEY || |
807 | ctx_p->cipher_mode == DRV_CIPHER_ECB); |
808 | |
809 | /* Process */ |
810 | if (req_ctx->dma_buf_type == CC_DMA_BUF_DLLI) { |
811 | dev_dbg(dev, " data params addr %pad length 0x%X\n" , |
812 | &sg_dma_address(src), nbytes); |
813 | dev_dbg(dev, " data params addr %pad length 0x%X\n" , |
814 | &sg_dma_address(dst), nbytes); |
815 | hw_desc_init(pdesc: &desc[*seq_size]); |
816 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_DLLI, sg_dma_address(src), |
817 | size: nbytes, NS_BIT); |
818 | set_dout_dlli(pdesc: &desc[*seq_size], sg_dma_address(dst), |
819 | size: nbytes, NS_BIT, last_ind: (!last_desc ? 0 : 1)); |
820 | if (last_desc) |
821 | set_queue_last_ind(drvdata: ctx_p->drvdata, pdesc: &desc[*seq_size]); |
822 | |
823 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
824 | (*seq_size)++; |
825 | } else { |
826 | hw_desc_init(pdesc: &desc[*seq_size]); |
827 | set_din_type(pdesc: &desc[*seq_size], dma_mode: DMA_MLLI, |
828 | addr: ctx_p->drvdata->mlli_sram_addr, |
829 | size: req_ctx->in_mlli_nents, NS_BIT); |
830 | if (req_ctx->out_nents == 0) { |
831 | dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n" , |
832 | ctx_p->drvdata->mlli_sram_addr, |
833 | ctx_p->drvdata->mlli_sram_addr); |
834 | set_dout_mlli(pdesc: &desc[*seq_size], |
835 | addr: ctx_p->drvdata->mlli_sram_addr, |
836 | size: req_ctx->in_mlli_nents, NS_BIT, |
837 | last_ind: (!last_desc ? 0 : 1)); |
838 | } else { |
839 | dev_dbg(dev, " din/dout params addr 0x%08X addr 0x%08X\n" , |
840 | ctx_p->drvdata->mlli_sram_addr, |
841 | ctx_p->drvdata->mlli_sram_addr + |
842 | (u32)LLI_ENTRY_BYTE_SIZE * req_ctx->in_nents); |
843 | set_dout_mlli(pdesc: &desc[*seq_size], |
844 | addr: (ctx_p->drvdata->mlli_sram_addr + |
845 | (LLI_ENTRY_BYTE_SIZE * |
846 | req_ctx->in_mlli_nents)), |
847 | size: req_ctx->out_mlli_nents, NS_BIT, |
848 | last_ind: (!last_desc ? 0 : 1)); |
849 | } |
850 | if (last_desc) |
851 | set_queue_last_ind(drvdata: ctx_p->drvdata, pdesc: &desc[*seq_size]); |
852 | |
853 | set_flow_mode(pdesc: &desc[*seq_size], mode: flow_mode); |
854 | (*seq_size)++; |
855 | } |
856 | } |
857 | |
858 | static void cc_cipher_complete(struct device *dev, void *cc_req, int err) |
859 | { |
860 | struct skcipher_request *req = (struct skcipher_request *)cc_req; |
861 | struct scatterlist *dst = req->dst; |
862 | struct scatterlist *src = req->src; |
863 | struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); |
864 | struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); |
865 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: sk_tfm); |
866 | |
867 | if (err != -EINPROGRESS) { |
868 | /* Not a BACKLOG notification */ |
869 | cc_unmap_cipher_request(dev, ctx: req_ctx, ivsize, src, dst); |
870 | memcpy(req->iv, req_ctx->iv, ivsize); |
871 | kfree_sensitive(objp: req_ctx->iv); |
872 | } |
873 | |
874 | skcipher_request_complete(req, err); |
875 | } |
876 | |
877 | static int cc_cipher_process(struct skcipher_request *req, |
878 | enum drv_crypto_direction direction) |
879 | { |
880 | struct crypto_skcipher *sk_tfm = crypto_skcipher_reqtfm(req); |
881 | struct crypto_tfm *tfm = crypto_skcipher_tfm(tfm: sk_tfm); |
882 | struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); |
883 | unsigned int ivsize = crypto_skcipher_ivsize(tfm: sk_tfm); |
884 | struct scatterlist *dst = req->dst; |
885 | struct scatterlist *src = req->src; |
886 | unsigned int nbytes = req->cryptlen; |
887 | void *iv = req->iv; |
888 | struct cc_cipher_ctx *ctx_p = crypto_tfm_ctx(tfm); |
889 | struct device *dev = drvdata_to_dev(drvdata: ctx_p->drvdata); |
890 | struct cc_hw_desc desc[MAX_SKCIPHER_SEQ_LEN]; |
891 | struct cc_crypto_req cc_req = {}; |
892 | int rc; |
893 | unsigned int seq_len = 0; |
894 | gfp_t flags = cc_gfp_flags(req: &req->base); |
895 | |
896 | dev_dbg(dev, "%s req=%p iv=%p nbytes=%d\n" , |
897 | ((direction == DRV_CRYPTO_DIRECTION_ENCRYPT) ? |
898 | "Encrypt" : "Decrypt" ), req, iv, nbytes); |
899 | |
900 | /* STAT_PHASE_0: Init and sanity checks */ |
901 | |
902 | if (validate_data_size(ctx_p, size: nbytes)) { |
903 | dev_dbg(dev, "Unsupported data size %d.\n" , nbytes); |
904 | rc = -EINVAL; |
905 | goto exit_process; |
906 | } |
907 | if (nbytes == 0) { |
908 | /* No data to process is valid */ |
909 | rc = 0; |
910 | goto exit_process; |
911 | } |
912 | |
913 | if (ctx_p->fallback_on) { |
914 | struct skcipher_request *subreq = skcipher_request_ctx(req); |
915 | |
916 | *subreq = *req; |
917 | skcipher_request_set_tfm(req: subreq, tfm: ctx_p->fallback_tfm); |
918 | if (direction == DRV_CRYPTO_DIRECTION_ENCRYPT) |
919 | return crypto_skcipher_encrypt(req: subreq); |
920 | else |
921 | return crypto_skcipher_decrypt(req: subreq); |
922 | } |
923 | |
924 | /* The IV we are handed may be allocated from the stack so |
925 | * we must copy it to a DMAable buffer before use. |
926 | */ |
927 | req_ctx->iv = kmemdup(p: iv, size: ivsize, gfp: flags); |
928 | if (!req_ctx->iv) { |
929 | rc = -ENOMEM; |
930 | goto exit_process; |
931 | } |
932 | |
933 | /* Setup request structure */ |
934 | cc_req.user_cb = cc_cipher_complete; |
935 | cc_req.user_arg = req; |
936 | |
937 | /* Setup CPP operation details */ |
938 | if (ctx_p->key_type == CC_POLICY_PROTECTED_KEY) { |
939 | cc_req.cpp.is_cpp = true; |
940 | cc_req.cpp.alg = ctx_p->cpp.alg; |
941 | cc_req.cpp.slot = ctx_p->cpp.slot; |
942 | } |
943 | |
944 | /* Setup request context */ |
945 | req_ctx->gen_ctx.op_type = direction; |
946 | |
947 | /* STAT_PHASE_1: Map buffers */ |
948 | |
949 | rc = cc_map_cipher_request(drvdata: ctx_p->drvdata, ctx: req_ctx, ivsize, nbytes, |
950 | info: req_ctx->iv, src, dst, flags); |
951 | if (rc) { |
952 | dev_err(dev, "map_request() failed\n" ); |
953 | goto exit_process; |
954 | } |
955 | |
956 | /* STAT_PHASE_2: Create sequence */ |
957 | |
958 | /* Setup state (IV) */ |
959 | cc_setup_state_desc(tfm, req_ctx, ivsize, nbytes, desc, seq_size: &seq_len); |
960 | /* Setup MLLI line, if needed */ |
961 | cc_setup_mlli_desc(tfm, req_ctx, dst, src, nbytes, areq: req, desc, seq_size: &seq_len); |
962 | /* Setup key */ |
963 | cc_setup_key_desc(tfm, req_ctx, nbytes, desc, seq_size: &seq_len); |
964 | /* Setup state (IV and XEX key) */ |
965 | cc_setup_xex_state_desc(tfm, req_ctx, ivsize, nbytes, desc, seq_size: &seq_len); |
966 | /* Data processing */ |
967 | cc_setup_flow_desc(tfm, req_ctx, dst, src, nbytes, desc, seq_size: &seq_len); |
968 | /* Read next IV */ |
969 | cc_setup_readiv_desc(tfm, req_ctx, ivsize, desc, seq_size: &seq_len); |
970 | |
971 | /* STAT_PHASE_3: Lock HW and push sequence */ |
972 | |
973 | rc = cc_send_request(drvdata: ctx_p->drvdata, cc_req: &cc_req, desc, len: seq_len, |
974 | req: &req->base); |
975 | if (rc != -EINPROGRESS && rc != -EBUSY) { |
976 | /* Failed to send the request or request completed |
977 | * synchronously |
978 | */ |
979 | cc_unmap_cipher_request(dev, ctx: req_ctx, ivsize, src, dst); |
980 | } |
981 | |
982 | exit_process: |
983 | if (rc != -EINPROGRESS && rc != -EBUSY) { |
984 | kfree_sensitive(objp: req_ctx->iv); |
985 | } |
986 | |
987 | return rc; |
988 | } |
989 | |
990 | static int cc_cipher_encrypt(struct skcipher_request *req) |
991 | { |
992 | struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); |
993 | |
994 | memset(req_ctx, 0, sizeof(*req_ctx)); |
995 | |
996 | return cc_cipher_process(req, direction: DRV_CRYPTO_DIRECTION_ENCRYPT); |
997 | } |
998 | |
999 | static int cc_cipher_decrypt(struct skcipher_request *req) |
1000 | { |
1001 | struct cipher_req_ctx *req_ctx = skcipher_request_ctx(req); |
1002 | |
1003 | memset(req_ctx, 0, sizeof(*req_ctx)); |
1004 | |
1005 | return cc_cipher_process(req, direction: DRV_CRYPTO_DIRECTION_DECRYPT); |
1006 | } |
1007 | |
1008 | /* Block cipher alg */ |
1009 | static const struct cc_alg_template skcipher_algs[] = { |
1010 | { |
1011 | .name = "xts(paes)" , |
1012 | .driver_name = "xts-paes-ccree" , |
1013 | .blocksize = 1, |
1014 | .template_skcipher = { |
1015 | .setkey = cc_cipher_sethkey, |
1016 | .encrypt = cc_cipher_encrypt, |
1017 | .decrypt = cc_cipher_decrypt, |
1018 | .min_keysize = CC_HW_KEY_SIZE, |
1019 | .max_keysize = CC_HW_KEY_SIZE, |
1020 | .ivsize = AES_BLOCK_SIZE, |
1021 | }, |
1022 | .cipher_mode = DRV_CIPHER_XTS, |
1023 | .flow_mode = S_DIN_to_AES, |
1024 | .min_hw_rev = CC_HW_REV_630, |
1025 | .std_body = CC_STD_NIST, |
1026 | .sec_func = true, |
1027 | }, |
1028 | { |
1029 | .name = "essiv(cbc(paes),sha256)" , |
1030 | .driver_name = "essiv-paes-ccree" , |
1031 | .blocksize = AES_BLOCK_SIZE, |
1032 | .template_skcipher = { |
1033 | .setkey = cc_cipher_sethkey, |
1034 | .encrypt = cc_cipher_encrypt, |
1035 | .decrypt = cc_cipher_decrypt, |
1036 | .min_keysize = CC_HW_KEY_SIZE, |
1037 | .max_keysize = CC_HW_KEY_SIZE, |
1038 | .ivsize = AES_BLOCK_SIZE, |
1039 | }, |
1040 | .cipher_mode = DRV_CIPHER_ESSIV, |
1041 | .flow_mode = S_DIN_to_AES, |
1042 | .min_hw_rev = CC_HW_REV_712, |
1043 | .std_body = CC_STD_NIST, |
1044 | .sec_func = true, |
1045 | }, |
1046 | { |
1047 | .name = "ecb(paes)" , |
1048 | .driver_name = "ecb-paes-ccree" , |
1049 | .blocksize = AES_BLOCK_SIZE, |
1050 | .template_skcipher = { |
1051 | .setkey = cc_cipher_sethkey, |
1052 | .encrypt = cc_cipher_encrypt, |
1053 | .decrypt = cc_cipher_decrypt, |
1054 | .min_keysize = CC_HW_KEY_SIZE, |
1055 | .max_keysize = CC_HW_KEY_SIZE, |
1056 | .ivsize = 0, |
1057 | }, |
1058 | .cipher_mode = DRV_CIPHER_ECB, |
1059 | .flow_mode = S_DIN_to_AES, |
1060 | .min_hw_rev = CC_HW_REV_712, |
1061 | .std_body = CC_STD_NIST, |
1062 | .sec_func = true, |
1063 | }, |
1064 | { |
1065 | .name = "cbc(paes)" , |
1066 | .driver_name = "cbc-paes-ccree" , |
1067 | .blocksize = AES_BLOCK_SIZE, |
1068 | .template_skcipher = { |
1069 | .setkey = cc_cipher_sethkey, |
1070 | .encrypt = cc_cipher_encrypt, |
1071 | .decrypt = cc_cipher_decrypt, |
1072 | .min_keysize = CC_HW_KEY_SIZE, |
1073 | .max_keysize = CC_HW_KEY_SIZE, |
1074 | .ivsize = AES_BLOCK_SIZE, |
1075 | }, |
1076 | .cipher_mode = DRV_CIPHER_CBC, |
1077 | .flow_mode = S_DIN_to_AES, |
1078 | .min_hw_rev = CC_HW_REV_712, |
1079 | .std_body = CC_STD_NIST, |
1080 | .sec_func = true, |
1081 | }, |
1082 | { |
1083 | .name = "cts(cbc(paes))" , |
1084 | .driver_name = "cts-cbc-paes-ccree" , |
1085 | .blocksize = AES_BLOCK_SIZE, |
1086 | .template_skcipher = { |
1087 | .setkey = cc_cipher_sethkey, |
1088 | .encrypt = cc_cipher_encrypt, |
1089 | .decrypt = cc_cipher_decrypt, |
1090 | .min_keysize = CC_HW_KEY_SIZE, |
1091 | .max_keysize = CC_HW_KEY_SIZE, |
1092 | .ivsize = AES_BLOCK_SIZE, |
1093 | }, |
1094 | .cipher_mode = DRV_CIPHER_CBC_CTS, |
1095 | .flow_mode = S_DIN_to_AES, |
1096 | .min_hw_rev = CC_HW_REV_712, |
1097 | .std_body = CC_STD_NIST, |
1098 | .sec_func = true, |
1099 | }, |
1100 | { |
1101 | .name = "ctr(paes)" , |
1102 | .driver_name = "ctr-paes-ccree" , |
1103 | .blocksize = 1, |
1104 | .template_skcipher = { |
1105 | .setkey = cc_cipher_sethkey, |
1106 | .encrypt = cc_cipher_encrypt, |
1107 | .decrypt = cc_cipher_decrypt, |
1108 | .min_keysize = CC_HW_KEY_SIZE, |
1109 | .max_keysize = CC_HW_KEY_SIZE, |
1110 | .ivsize = AES_BLOCK_SIZE, |
1111 | }, |
1112 | .cipher_mode = DRV_CIPHER_CTR, |
1113 | .flow_mode = S_DIN_to_AES, |
1114 | .min_hw_rev = CC_HW_REV_712, |
1115 | .std_body = CC_STD_NIST, |
1116 | .sec_func = true, |
1117 | }, |
1118 | { |
1119 | /* See https://www.mail-archive.com/linux-crypto@vger.kernel.org/msg40576.html |
1120 | * for the reason why this differs from the generic |
1121 | * implementation. |
1122 | */ |
1123 | .name = "xts(aes)" , |
1124 | .driver_name = "xts-aes-ccree" , |
1125 | .blocksize = 1, |
1126 | .template_skcipher = { |
1127 | .setkey = cc_cipher_setkey, |
1128 | .encrypt = cc_cipher_encrypt, |
1129 | .decrypt = cc_cipher_decrypt, |
1130 | .min_keysize = AES_MIN_KEY_SIZE * 2, |
1131 | .max_keysize = AES_MAX_KEY_SIZE * 2, |
1132 | .ivsize = AES_BLOCK_SIZE, |
1133 | }, |
1134 | .cipher_mode = DRV_CIPHER_XTS, |
1135 | .flow_mode = S_DIN_to_AES, |
1136 | .min_hw_rev = CC_HW_REV_630, |
1137 | .std_body = CC_STD_NIST, |
1138 | }, |
1139 | { |
1140 | .name = "essiv(cbc(aes),sha256)" , |
1141 | .driver_name = "essiv-aes-ccree" , |
1142 | .blocksize = AES_BLOCK_SIZE, |
1143 | .template_skcipher = { |
1144 | .setkey = cc_cipher_setkey, |
1145 | .encrypt = cc_cipher_encrypt, |
1146 | .decrypt = cc_cipher_decrypt, |
1147 | .min_keysize = AES_MIN_KEY_SIZE, |
1148 | .max_keysize = AES_MAX_KEY_SIZE, |
1149 | .ivsize = AES_BLOCK_SIZE, |
1150 | }, |
1151 | .cipher_mode = DRV_CIPHER_ESSIV, |
1152 | .flow_mode = S_DIN_to_AES, |
1153 | .min_hw_rev = CC_HW_REV_712, |
1154 | .std_body = CC_STD_NIST, |
1155 | }, |
1156 | { |
1157 | .name = "ecb(aes)" , |
1158 | .driver_name = "ecb-aes-ccree" , |
1159 | .blocksize = AES_BLOCK_SIZE, |
1160 | .template_skcipher = { |
1161 | .setkey = cc_cipher_setkey, |
1162 | .encrypt = cc_cipher_encrypt, |
1163 | .decrypt = cc_cipher_decrypt, |
1164 | .min_keysize = AES_MIN_KEY_SIZE, |
1165 | .max_keysize = AES_MAX_KEY_SIZE, |
1166 | .ivsize = 0, |
1167 | }, |
1168 | .cipher_mode = DRV_CIPHER_ECB, |
1169 | .flow_mode = S_DIN_to_AES, |
1170 | .min_hw_rev = CC_HW_REV_630, |
1171 | .std_body = CC_STD_NIST, |
1172 | }, |
1173 | { |
1174 | .name = "cbc(aes)" , |
1175 | .driver_name = "cbc-aes-ccree" , |
1176 | .blocksize = AES_BLOCK_SIZE, |
1177 | .template_skcipher = { |
1178 | .setkey = cc_cipher_setkey, |
1179 | .encrypt = cc_cipher_encrypt, |
1180 | .decrypt = cc_cipher_decrypt, |
1181 | .min_keysize = AES_MIN_KEY_SIZE, |
1182 | .max_keysize = AES_MAX_KEY_SIZE, |
1183 | .ivsize = AES_BLOCK_SIZE, |
1184 | }, |
1185 | .cipher_mode = DRV_CIPHER_CBC, |
1186 | .flow_mode = S_DIN_to_AES, |
1187 | .min_hw_rev = CC_HW_REV_630, |
1188 | .std_body = CC_STD_NIST, |
1189 | }, |
1190 | { |
1191 | .name = "cts(cbc(aes))" , |
1192 | .driver_name = "cts-cbc-aes-ccree" , |
1193 | .blocksize = AES_BLOCK_SIZE, |
1194 | .template_skcipher = { |
1195 | .setkey = cc_cipher_setkey, |
1196 | .encrypt = cc_cipher_encrypt, |
1197 | .decrypt = cc_cipher_decrypt, |
1198 | .min_keysize = AES_MIN_KEY_SIZE, |
1199 | .max_keysize = AES_MAX_KEY_SIZE, |
1200 | .ivsize = AES_BLOCK_SIZE, |
1201 | }, |
1202 | .cipher_mode = DRV_CIPHER_CBC_CTS, |
1203 | .flow_mode = S_DIN_to_AES, |
1204 | .min_hw_rev = CC_HW_REV_630, |
1205 | .std_body = CC_STD_NIST, |
1206 | }, |
1207 | { |
1208 | .name = "ctr(aes)" , |
1209 | .driver_name = "ctr-aes-ccree" , |
1210 | .blocksize = 1, |
1211 | .template_skcipher = { |
1212 | .setkey = cc_cipher_setkey, |
1213 | .encrypt = cc_cipher_encrypt, |
1214 | .decrypt = cc_cipher_decrypt, |
1215 | .min_keysize = AES_MIN_KEY_SIZE, |
1216 | .max_keysize = AES_MAX_KEY_SIZE, |
1217 | .ivsize = AES_BLOCK_SIZE, |
1218 | }, |
1219 | .cipher_mode = DRV_CIPHER_CTR, |
1220 | .flow_mode = S_DIN_to_AES, |
1221 | .min_hw_rev = CC_HW_REV_630, |
1222 | .std_body = CC_STD_NIST, |
1223 | }, |
1224 | { |
1225 | .name = "cbc(des3_ede)" , |
1226 | .driver_name = "cbc-3des-ccree" , |
1227 | .blocksize = DES3_EDE_BLOCK_SIZE, |
1228 | .template_skcipher = { |
1229 | .setkey = cc_cipher_setkey, |
1230 | .encrypt = cc_cipher_encrypt, |
1231 | .decrypt = cc_cipher_decrypt, |
1232 | .min_keysize = DES3_EDE_KEY_SIZE, |
1233 | .max_keysize = DES3_EDE_KEY_SIZE, |
1234 | .ivsize = DES3_EDE_BLOCK_SIZE, |
1235 | }, |
1236 | .cipher_mode = DRV_CIPHER_CBC, |
1237 | .flow_mode = S_DIN_to_DES, |
1238 | .min_hw_rev = CC_HW_REV_630, |
1239 | .std_body = CC_STD_NIST, |
1240 | }, |
1241 | { |
1242 | .name = "ecb(des3_ede)" , |
1243 | .driver_name = "ecb-3des-ccree" , |
1244 | .blocksize = DES3_EDE_BLOCK_SIZE, |
1245 | .template_skcipher = { |
1246 | .setkey = cc_cipher_setkey, |
1247 | .encrypt = cc_cipher_encrypt, |
1248 | .decrypt = cc_cipher_decrypt, |
1249 | .min_keysize = DES3_EDE_KEY_SIZE, |
1250 | .max_keysize = DES3_EDE_KEY_SIZE, |
1251 | .ivsize = 0, |
1252 | }, |
1253 | .cipher_mode = DRV_CIPHER_ECB, |
1254 | .flow_mode = S_DIN_to_DES, |
1255 | .min_hw_rev = CC_HW_REV_630, |
1256 | .std_body = CC_STD_NIST, |
1257 | }, |
1258 | { |
1259 | .name = "cbc(des)" , |
1260 | .driver_name = "cbc-des-ccree" , |
1261 | .blocksize = DES_BLOCK_SIZE, |
1262 | .template_skcipher = { |
1263 | .setkey = cc_cipher_setkey, |
1264 | .encrypt = cc_cipher_encrypt, |
1265 | .decrypt = cc_cipher_decrypt, |
1266 | .min_keysize = DES_KEY_SIZE, |
1267 | .max_keysize = DES_KEY_SIZE, |
1268 | .ivsize = DES_BLOCK_SIZE, |
1269 | }, |
1270 | .cipher_mode = DRV_CIPHER_CBC, |
1271 | .flow_mode = S_DIN_to_DES, |
1272 | .min_hw_rev = CC_HW_REV_630, |
1273 | .std_body = CC_STD_NIST, |
1274 | }, |
1275 | { |
1276 | .name = "ecb(des)" , |
1277 | .driver_name = "ecb-des-ccree" , |
1278 | .blocksize = DES_BLOCK_SIZE, |
1279 | .template_skcipher = { |
1280 | .setkey = cc_cipher_setkey, |
1281 | .encrypt = cc_cipher_encrypt, |
1282 | .decrypt = cc_cipher_decrypt, |
1283 | .min_keysize = DES_KEY_SIZE, |
1284 | .max_keysize = DES_KEY_SIZE, |
1285 | .ivsize = 0, |
1286 | }, |
1287 | .cipher_mode = DRV_CIPHER_ECB, |
1288 | .flow_mode = S_DIN_to_DES, |
1289 | .min_hw_rev = CC_HW_REV_630, |
1290 | .std_body = CC_STD_NIST, |
1291 | }, |
1292 | { |
1293 | .name = "cbc(sm4)" , |
1294 | .driver_name = "cbc-sm4-ccree" , |
1295 | .blocksize = SM4_BLOCK_SIZE, |
1296 | .template_skcipher = { |
1297 | .setkey = cc_cipher_setkey, |
1298 | .encrypt = cc_cipher_encrypt, |
1299 | .decrypt = cc_cipher_decrypt, |
1300 | .min_keysize = SM4_KEY_SIZE, |
1301 | .max_keysize = SM4_KEY_SIZE, |
1302 | .ivsize = SM4_BLOCK_SIZE, |
1303 | }, |
1304 | .cipher_mode = DRV_CIPHER_CBC, |
1305 | .flow_mode = S_DIN_to_SM4, |
1306 | .min_hw_rev = CC_HW_REV_713, |
1307 | .std_body = CC_STD_OSCCA, |
1308 | }, |
1309 | { |
1310 | .name = "ecb(sm4)" , |
1311 | .driver_name = "ecb-sm4-ccree" , |
1312 | .blocksize = SM4_BLOCK_SIZE, |
1313 | .template_skcipher = { |
1314 | .setkey = cc_cipher_setkey, |
1315 | .encrypt = cc_cipher_encrypt, |
1316 | .decrypt = cc_cipher_decrypt, |
1317 | .min_keysize = SM4_KEY_SIZE, |
1318 | .max_keysize = SM4_KEY_SIZE, |
1319 | .ivsize = 0, |
1320 | }, |
1321 | .cipher_mode = DRV_CIPHER_ECB, |
1322 | .flow_mode = S_DIN_to_SM4, |
1323 | .min_hw_rev = CC_HW_REV_713, |
1324 | .std_body = CC_STD_OSCCA, |
1325 | }, |
1326 | { |
1327 | .name = "ctr(sm4)" , |
1328 | .driver_name = "ctr-sm4-ccree" , |
1329 | .blocksize = 1, |
1330 | .template_skcipher = { |
1331 | .setkey = cc_cipher_setkey, |
1332 | .encrypt = cc_cipher_encrypt, |
1333 | .decrypt = cc_cipher_decrypt, |
1334 | .min_keysize = SM4_KEY_SIZE, |
1335 | .max_keysize = SM4_KEY_SIZE, |
1336 | .ivsize = SM4_BLOCK_SIZE, |
1337 | }, |
1338 | .cipher_mode = DRV_CIPHER_CTR, |
1339 | .flow_mode = S_DIN_to_SM4, |
1340 | .min_hw_rev = CC_HW_REV_713, |
1341 | .std_body = CC_STD_OSCCA, |
1342 | }, |
1343 | { |
1344 | .name = "cbc(psm4)" , |
1345 | .driver_name = "cbc-psm4-ccree" , |
1346 | .blocksize = SM4_BLOCK_SIZE, |
1347 | .template_skcipher = { |
1348 | .setkey = cc_cipher_sethkey, |
1349 | .encrypt = cc_cipher_encrypt, |
1350 | .decrypt = cc_cipher_decrypt, |
1351 | .min_keysize = CC_HW_KEY_SIZE, |
1352 | .max_keysize = CC_HW_KEY_SIZE, |
1353 | .ivsize = SM4_BLOCK_SIZE, |
1354 | }, |
1355 | .cipher_mode = DRV_CIPHER_CBC, |
1356 | .flow_mode = S_DIN_to_SM4, |
1357 | .min_hw_rev = CC_HW_REV_713, |
1358 | .std_body = CC_STD_OSCCA, |
1359 | .sec_func = true, |
1360 | }, |
1361 | { |
1362 | .name = "ctr(psm4)" , |
1363 | .driver_name = "ctr-psm4-ccree" , |
1364 | .blocksize = SM4_BLOCK_SIZE, |
1365 | .template_skcipher = { |
1366 | .setkey = cc_cipher_sethkey, |
1367 | .encrypt = cc_cipher_encrypt, |
1368 | .decrypt = cc_cipher_decrypt, |
1369 | .min_keysize = CC_HW_KEY_SIZE, |
1370 | .max_keysize = CC_HW_KEY_SIZE, |
1371 | .ivsize = SM4_BLOCK_SIZE, |
1372 | }, |
1373 | .cipher_mode = DRV_CIPHER_CTR, |
1374 | .flow_mode = S_DIN_to_SM4, |
1375 | .min_hw_rev = CC_HW_REV_713, |
1376 | .std_body = CC_STD_OSCCA, |
1377 | .sec_func = true, |
1378 | }, |
1379 | }; |
1380 | |
1381 | static struct cc_crypto_alg *cc_create_alg(const struct cc_alg_template *tmpl, |
1382 | struct device *dev) |
1383 | { |
1384 | struct cc_crypto_alg *t_alg; |
1385 | struct skcipher_alg *alg; |
1386 | |
1387 | t_alg = devm_kzalloc(dev, size: sizeof(*t_alg), GFP_KERNEL); |
1388 | if (!t_alg) |
1389 | return ERR_PTR(error: -ENOMEM); |
1390 | |
1391 | alg = &t_alg->skcipher_alg; |
1392 | |
1393 | memcpy(alg, &tmpl->template_skcipher, sizeof(*alg)); |
1394 | |
1395 | if (snprintf(buf: alg->base.cra_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , |
1396 | tmpl->name) >= CRYPTO_MAX_ALG_NAME) |
1397 | return ERR_PTR(error: -EINVAL); |
1398 | if (snprintf(buf: alg->base.cra_driver_name, CRYPTO_MAX_ALG_NAME, fmt: "%s" , |
1399 | tmpl->driver_name) >= CRYPTO_MAX_ALG_NAME) |
1400 | return ERR_PTR(error: -EINVAL); |
1401 | |
1402 | alg->base.cra_module = THIS_MODULE; |
1403 | alg->base.cra_priority = CC_CRA_PRIO; |
1404 | alg->base.cra_blocksize = tmpl->blocksize; |
1405 | alg->base.cra_alignmask = 0; |
1406 | alg->base.cra_ctxsize = sizeof(struct cc_cipher_ctx); |
1407 | |
1408 | alg->base.cra_init = cc_cipher_init; |
1409 | alg->base.cra_exit = cc_cipher_exit; |
1410 | alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; |
1411 | |
1412 | t_alg->cipher_mode = tmpl->cipher_mode; |
1413 | t_alg->flow_mode = tmpl->flow_mode; |
1414 | |
1415 | return t_alg; |
1416 | } |
1417 | |
1418 | int cc_cipher_free(struct cc_drvdata *drvdata) |
1419 | { |
1420 | struct cc_crypto_alg *t_alg, *n; |
1421 | |
1422 | /* Remove registered algs */ |
1423 | list_for_each_entry_safe(t_alg, n, &drvdata->alg_list, entry) { |
1424 | crypto_unregister_skcipher(alg: &t_alg->skcipher_alg); |
1425 | list_del(entry: &t_alg->entry); |
1426 | } |
1427 | return 0; |
1428 | } |
1429 | |
1430 | int cc_cipher_alloc(struct cc_drvdata *drvdata) |
1431 | { |
1432 | struct cc_crypto_alg *t_alg; |
1433 | struct device *dev = drvdata_to_dev(drvdata); |
1434 | int rc = -ENOMEM; |
1435 | int alg; |
1436 | |
1437 | INIT_LIST_HEAD(list: &drvdata->alg_list); |
1438 | |
1439 | /* Linux crypto */ |
1440 | dev_dbg(dev, "Number of algorithms = %zu\n" , |
1441 | ARRAY_SIZE(skcipher_algs)); |
1442 | for (alg = 0; alg < ARRAY_SIZE(skcipher_algs); alg++) { |
1443 | if ((skcipher_algs[alg].min_hw_rev > drvdata->hw_rev) || |
1444 | !(drvdata->std_bodies & skcipher_algs[alg].std_body) || |
1445 | (drvdata->sec_disabled && skcipher_algs[alg].sec_func)) |
1446 | continue; |
1447 | |
1448 | dev_dbg(dev, "creating %s\n" , skcipher_algs[alg].driver_name); |
1449 | t_alg = cc_create_alg(tmpl: &skcipher_algs[alg], dev); |
1450 | if (IS_ERR(ptr: t_alg)) { |
1451 | rc = PTR_ERR(ptr: t_alg); |
1452 | dev_err(dev, "%s alg allocation failed\n" , |
1453 | skcipher_algs[alg].driver_name); |
1454 | goto fail0; |
1455 | } |
1456 | t_alg->drvdata = drvdata; |
1457 | |
1458 | dev_dbg(dev, "registering %s\n" , |
1459 | skcipher_algs[alg].driver_name); |
1460 | rc = crypto_register_skcipher(alg: &t_alg->skcipher_alg); |
1461 | dev_dbg(dev, "%s alg registration rc = %x\n" , |
1462 | t_alg->skcipher_alg.base.cra_driver_name, rc); |
1463 | if (rc) { |
1464 | dev_err(dev, "%s alg registration failed\n" , |
1465 | t_alg->skcipher_alg.base.cra_driver_name); |
1466 | goto fail0; |
1467 | } |
1468 | |
1469 | list_add_tail(new: &t_alg->entry, head: &drvdata->alg_list); |
1470 | dev_dbg(dev, "Registered %s\n" , |
1471 | t_alg->skcipher_alg.base.cra_driver_name); |
1472 | } |
1473 | return 0; |
1474 | |
1475 | fail0: |
1476 | cc_cipher_free(drvdata); |
1477 | return rc; |
1478 | } |
1479 | |