1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * sl3516-ce-cipher.c - hardware cryptographic offloader for Storlink SL3516 SoC |
4 | * |
5 | * Copyright (C) 2021 Corentin LABBE <clabbe@baylibre.com> |
6 | * |
7 | * This file adds support for AES cipher with 128,192,256 bits keysize in |
8 | * ECB mode. |
9 | */ |
10 | |
11 | #include <crypto/engine.h> |
12 | #include <crypto/internal/skcipher.h> |
13 | #include <crypto/scatterwalk.h> |
14 | #include <linux/dma-mapping.h> |
15 | #include <linux/delay.h> |
16 | #include <linux/err.h> |
17 | #include <linux/io.h> |
18 | #include <linux/kernel.h> |
19 | #include <linux/pm_runtime.h> |
20 | #include <linux/slab.h> |
21 | #include <linux/string.h> |
22 | #include "sl3516-ce.h" |
23 | |
24 | /* sl3516_ce_need_fallback - check if a request can be handled by the CE */ |
25 | static bool sl3516_ce_need_fallback(struct skcipher_request *areq) |
26 | { |
27 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
28 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
29 | struct sl3516_ce_dev *ce = op->ce; |
30 | struct scatterlist *in_sg; |
31 | struct scatterlist *out_sg; |
32 | struct scatterlist *sg; |
33 | |
34 | if (areq->cryptlen == 0 || areq->cryptlen % 16) { |
35 | ce->fallback_mod16++; |
36 | return true; |
37 | } |
38 | |
39 | /* |
40 | * check if we have enough descriptors for TX |
41 | * Note: TX need one control desc for each SG |
42 | */ |
43 | if (sg_nents(sg: areq->src) > MAXDESC / 2) { |
44 | ce->fallback_sg_count_tx++; |
45 | return true; |
46 | } |
47 | /* check if we have enough descriptors for RX */ |
48 | if (sg_nents(sg: areq->dst) > MAXDESC) { |
49 | ce->fallback_sg_count_rx++; |
50 | return true; |
51 | } |
52 | |
53 | sg = areq->src; |
54 | while (sg) { |
55 | if ((sg->length % 16) != 0) { |
56 | ce->fallback_mod16++; |
57 | return true; |
58 | } |
59 | if ((sg_dma_len(sg) % 16) != 0) { |
60 | ce->fallback_mod16++; |
61 | return true; |
62 | } |
63 | if (!IS_ALIGNED(sg->offset, 16)) { |
64 | ce->fallback_align16++; |
65 | return true; |
66 | } |
67 | sg = sg_next(sg); |
68 | } |
69 | sg = areq->dst; |
70 | while (sg) { |
71 | if ((sg->length % 16) != 0) { |
72 | ce->fallback_mod16++; |
73 | return true; |
74 | } |
75 | if ((sg_dma_len(sg) % 16) != 0) { |
76 | ce->fallback_mod16++; |
77 | return true; |
78 | } |
79 | if (!IS_ALIGNED(sg->offset, 16)) { |
80 | ce->fallback_align16++; |
81 | return true; |
82 | } |
83 | sg = sg_next(sg); |
84 | } |
85 | |
86 | /* need same numbers of SG (with same length) for source and destination */ |
87 | in_sg = areq->src; |
88 | out_sg = areq->dst; |
89 | while (in_sg && out_sg) { |
90 | if (in_sg->length != out_sg->length) { |
91 | ce->fallback_not_same_len++; |
92 | return true; |
93 | } |
94 | in_sg = sg_next(in_sg); |
95 | out_sg = sg_next(out_sg); |
96 | } |
97 | if (in_sg || out_sg) |
98 | return true; |
99 | |
100 | return false; |
101 | } |
102 | |
103 | static int sl3516_ce_cipher_fallback(struct skcipher_request *areq) |
104 | { |
105 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
106 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
107 | struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
108 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
109 | struct sl3516_ce_alg_template *algt; |
110 | int err; |
111 | |
112 | algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); |
113 | algt->stat_fb++; |
114 | |
115 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: op->fallback_tfm); |
116 | skcipher_request_set_callback(req: &rctx->fallback_req, flags: areq->base.flags, |
117 | compl: areq->base.complete, data: areq->base.data); |
118 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: areq->src, dst: areq->dst, |
119 | cryptlen: areq->cryptlen, iv: areq->iv); |
120 | if (rctx->op_dir == CE_DECRYPTION) |
121 | err = crypto_skcipher_decrypt(req: &rctx->fallback_req); |
122 | else |
123 | err = crypto_skcipher_encrypt(req: &rctx->fallback_req); |
124 | return err; |
125 | } |
126 | |
127 | static int sl3516_ce_cipher(struct skcipher_request *areq) |
128 | { |
129 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
130 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
131 | struct sl3516_ce_dev *ce = op->ce; |
132 | struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
133 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
134 | struct sl3516_ce_alg_template *algt; |
135 | struct scatterlist *sg; |
136 | unsigned int todo, len; |
137 | struct pkt_control_ecb *ecb; |
138 | int nr_sgs = 0; |
139 | int nr_sgd = 0; |
140 | int err = 0; |
141 | int i; |
142 | |
143 | algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); |
144 | |
145 | dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n" , __func__, |
146 | crypto_tfm_alg_name(areq->base.tfm), |
147 | areq->cryptlen, |
148 | rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), |
149 | op->keylen); |
150 | |
151 | algt->stat_req++; |
152 | |
153 | if (areq->src == areq->dst) { |
154 | nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), |
155 | DMA_BIDIRECTIONAL); |
156 | if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { |
157 | dev_err(ce->dev, "Invalid sg number %d\n" , nr_sgs); |
158 | err = -EINVAL; |
159 | goto theend; |
160 | } |
161 | nr_sgd = nr_sgs; |
162 | } else { |
163 | nr_sgs = dma_map_sg(ce->dev, areq->src, sg_nents(areq->src), |
164 | DMA_TO_DEVICE); |
165 | if (nr_sgs <= 0 || nr_sgs > MAXDESC / 2) { |
166 | dev_err(ce->dev, "Invalid sg number %d\n" , nr_sgs); |
167 | err = -EINVAL; |
168 | goto theend; |
169 | } |
170 | nr_sgd = dma_map_sg(ce->dev, areq->dst, sg_nents(areq->dst), |
171 | DMA_FROM_DEVICE); |
172 | if (nr_sgd <= 0 || nr_sgd > MAXDESC) { |
173 | dev_err(ce->dev, "Invalid sg number %d\n" , nr_sgd); |
174 | err = -EINVAL; |
175 | goto theend_sgs; |
176 | } |
177 | } |
178 | |
179 | len = areq->cryptlen; |
180 | i = 0; |
181 | sg = areq->src; |
182 | while (i < nr_sgs && sg && len) { |
183 | if (sg_dma_len(sg) == 0) |
184 | goto sgs_next; |
185 | rctx->t_src[i].addr = sg_dma_address(sg); |
186 | todo = min(len, sg_dma_len(sg)); |
187 | rctx->t_src[i].len = todo; |
188 | dev_dbg(ce->dev, "%s total=%u SGS(%d %u off=%d) todo=%u\n" , __func__, |
189 | areq->cryptlen, i, rctx->t_src[i].len, sg->offset, todo); |
190 | len -= todo; |
191 | i++; |
192 | sgs_next: |
193 | sg = sg_next(sg); |
194 | } |
195 | if (len > 0) { |
196 | dev_err(ce->dev, "remaining len %d/%u nr_sgs=%d\n" , len, areq->cryptlen, nr_sgs); |
197 | err = -EINVAL; |
198 | goto theend_sgs; |
199 | } |
200 | |
201 | len = areq->cryptlen; |
202 | i = 0; |
203 | sg = areq->dst; |
204 | while (i < nr_sgd && sg && len) { |
205 | if (sg_dma_len(sg) == 0) |
206 | goto sgd_next; |
207 | rctx->t_dst[i].addr = sg_dma_address(sg); |
208 | todo = min(len, sg_dma_len(sg)); |
209 | rctx->t_dst[i].len = todo; |
210 | dev_dbg(ce->dev, "%s total=%u SGD(%d %u off=%d) todo=%u\n" , __func__, |
211 | areq->cryptlen, i, rctx->t_dst[i].len, sg->offset, todo); |
212 | len -= todo; |
213 | i++; |
214 | |
215 | sgd_next: |
216 | sg = sg_next(sg); |
217 | } |
218 | if (len > 0) { |
219 | dev_err(ce->dev, "remaining len %d\n" , len); |
220 | err = -EINVAL; |
221 | goto theend_sgs; |
222 | } |
223 | |
224 | switch (algt->mode) { |
225 | case ECB_AES: |
226 | rctx->pctrllen = sizeof(struct pkt_control_ecb); |
227 | ecb = (struct pkt_control_ecb *)ce->pctrl; |
228 | |
229 | rctx->tqflag = TQ0_TYPE_CTRL; |
230 | rctx->tqflag |= TQ1_CIPHER; |
231 | ecb->control.op_mode = rctx->op_dir; |
232 | ecb->control.cipher_algorithm = ECB_AES; |
233 | ecb->cipher.header_len = 0; |
234 | ecb->cipher.algorithm_len = areq->cryptlen; |
235 | cpu_to_be32_array(dst: (__be32 *)ecb->key, src: (u32 *)op->key, len: op->keylen / 4); |
236 | rctx->h = &ecb->cipher; |
237 | |
238 | rctx->tqflag |= TQ4_KEY0; |
239 | rctx->tqflag |= TQ5_KEY4; |
240 | rctx->tqflag |= TQ6_KEY6; |
241 | ecb->control.aesnk = op->keylen / 4; |
242 | break; |
243 | } |
244 | |
245 | rctx->nr_sgs = nr_sgs; |
246 | rctx->nr_sgd = nr_sgd; |
247 | err = sl3516_ce_run_task(ce, rctx, name: crypto_tfm_alg_name(tfm: areq->base.tfm)); |
248 | |
249 | theend_sgs: |
250 | if (areq->src == areq->dst) { |
251 | dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), |
252 | DMA_BIDIRECTIONAL); |
253 | } else { |
254 | dma_unmap_sg(ce->dev, areq->src, sg_nents(areq->src), |
255 | DMA_TO_DEVICE); |
256 | dma_unmap_sg(ce->dev, areq->dst, sg_nents(areq->dst), |
257 | DMA_FROM_DEVICE); |
258 | } |
259 | |
260 | theend: |
261 | |
262 | return err; |
263 | } |
264 | |
265 | int sl3516_ce_handle_cipher_request(struct crypto_engine *engine, void *areq) |
266 | { |
267 | int err; |
268 | struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); |
269 | |
270 | err = sl3516_ce_cipher(areq: breq); |
271 | local_bh_disable(); |
272 | crypto_finalize_skcipher_request(engine, req: breq, err); |
273 | local_bh_enable(); |
274 | |
275 | return 0; |
276 | } |
277 | |
278 | int sl3516_ce_skdecrypt(struct skcipher_request *areq) |
279 | { |
280 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
281 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
282 | struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
283 | struct crypto_engine *engine; |
284 | |
285 | memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); |
286 | rctx->op_dir = CE_DECRYPTION; |
287 | |
288 | if (sl3516_ce_need_fallback(areq)) |
289 | return sl3516_ce_cipher_fallback(areq); |
290 | |
291 | engine = op->ce->engine; |
292 | |
293 | return crypto_transfer_skcipher_request_to_engine(engine, req: areq); |
294 | } |
295 | |
296 | int sl3516_ce_skencrypt(struct skcipher_request *areq) |
297 | { |
298 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
299 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
300 | struct sl3516_ce_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
301 | struct crypto_engine *engine; |
302 | |
303 | memset(rctx, 0, sizeof(struct sl3516_ce_cipher_req_ctx)); |
304 | rctx->op_dir = CE_ENCRYPTION; |
305 | |
306 | if (sl3516_ce_need_fallback(areq)) |
307 | return sl3516_ce_cipher_fallback(areq); |
308 | |
309 | engine = op->ce->engine; |
310 | |
311 | return crypto_transfer_skcipher_request_to_engine(engine, req: areq); |
312 | } |
313 | |
314 | int sl3516_ce_cipher_init(struct crypto_tfm *tfm) |
315 | { |
316 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); |
317 | struct sl3516_ce_alg_template *algt; |
318 | const char *name = crypto_tfm_alg_name(tfm); |
319 | struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); |
320 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm: sktfm); |
321 | int err; |
322 | |
323 | memset(op, 0, sizeof(struct sl3516_ce_cipher_tfm_ctx)); |
324 | |
325 | algt = container_of(alg, struct sl3516_ce_alg_template, alg.skcipher.base); |
326 | op->ce = algt->ce; |
327 | |
328 | op->fallback_tfm = crypto_alloc_skcipher(alg_name: name, type: 0, CRYPTO_ALG_NEED_FALLBACK); |
329 | if (IS_ERR(ptr: op->fallback_tfm)) { |
330 | dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n" , |
331 | name, PTR_ERR(op->fallback_tfm)); |
332 | return PTR_ERR(ptr: op->fallback_tfm); |
333 | } |
334 | |
335 | crypto_skcipher_set_reqsize(skcipher: sktfm, reqsize: sizeof(struct sl3516_ce_cipher_req_ctx) + |
336 | crypto_skcipher_reqsize(tfm: op->fallback_tfm)); |
337 | |
338 | dev_info(op->ce->dev, "Fallback for %s is %s\n" , |
339 | crypto_tfm_alg_driver_name(&sktfm->base), |
340 | crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm))); |
341 | |
342 | err = pm_runtime_get_sync(dev: op->ce->dev); |
343 | if (err < 0) |
344 | goto error_pm; |
345 | |
346 | return 0; |
347 | error_pm: |
348 | pm_runtime_put_noidle(dev: op->ce->dev); |
349 | crypto_free_skcipher(tfm: op->fallback_tfm); |
350 | return err; |
351 | } |
352 | |
353 | void sl3516_ce_cipher_exit(struct crypto_tfm *tfm) |
354 | { |
355 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); |
356 | |
357 | kfree_sensitive(objp: op->key); |
358 | crypto_free_skcipher(tfm: op->fallback_tfm); |
359 | pm_runtime_put_sync_suspend(dev: op->ce->dev); |
360 | } |
361 | |
362 | int sl3516_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, |
363 | unsigned int keylen) |
364 | { |
365 | struct sl3516_ce_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
366 | struct sl3516_ce_dev *ce = op->ce; |
367 | |
368 | switch (keylen) { |
369 | case 128 / 8: |
370 | break; |
371 | case 192 / 8: |
372 | break; |
373 | case 256 / 8: |
374 | break; |
375 | default: |
376 | dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n" , keylen); |
377 | return -EINVAL; |
378 | } |
379 | kfree_sensitive(objp: op->key); |
380 | op->keylen = keylen; |
381 | op->key = kmemdup(p: key, size: keylen, GFP_KERNEL | GFP_DMA); |
382 | if (!op->key) |
383 | return -ENOMEM; |
384 | |
385 | crypto_skcipher_clear_flags(tfm: op->fallback_tfm, CRYPTO_TFM_REQ_MASK); |
386 | crypto_skcipher_set_flags(tfm: op->fallback_tfm, flags: tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); |
387 | |
388 | return crypto_skcipher_setkey(tfm: op->fallback_tfm, key, keylen); |
389 | } |
390 | |