1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * amlogic-cipher.c - hardware cryptographic offloader for Amlogic GXL SoC |
4 | * |
5 | * Copyright (C) 2018-2019 Corentin LABBE <clabbe@baylibre.com> |
6 | * |
7 | * This file add support for AES cipher with 128,192,256 bits keysize in |
8 | * CBC and ECB mode. |
9 | */ |
10 | |
11 | #include <linux/crypto.h> |
12 | #include <linux/delay.h> |
13 | #include <linux/io.h> |
14 | #include <crypto/scatterwalk.h> |
15 | #include <linux/scatterlist.h> |
16 | #include <linux/dma-mapping.h> |
17 | #include <crypto/internal/skcipher.h> |
18 | #include "amlogic-gxl.h" |
19 | |
20 | static int get_engine_number(struct meson_dev *mc) |
21 | { |
22 | return atomic_inc_return(v: &mc->flow) % MAXFLOW; |
23 | } |
24 | |
25 | static bool meson_cipher_need_fallback(struct skcipher_request *areq) |
26 | { |
27 | struct scatterlist *src_sg = areq->src; |
28 | struct scatterlist *dst_sg = areq->dst; |
29 | |
30 | if (areq->cryptlen == 0) |
31 | return true; |
32 | |
33 | if (sg_nents(sg: src_sg) != sg_nents(sg: dst_sg)) |
34 | return true; |
35 | |
36 | /* KEY/IV descriptors use 3 desc */ |
37 | if (sg_nents(sg: src_sg) > MAXDESC - 3 || sg_nents(sg: dst_sg) > MAXDESC - 3) |
38 | return true; |
39 | |
40 | while (src_sg && dst_sg) { |
41 | if ((src_sg->length % 16) != 0) |
42 | return true; |
43 | if ((dst_sg->length % 16) != 0) |
44 | return true; |
45 | if (src_sg->length != dst_sg->length) |
46 | return true; |
47 | if (!IS_ALIGNED(src_sg->offset, sizeof(u32))) |
48 | return true; |
49 | if (!IS_ALIGNED(dst_sg->offset, sizeof(u32))) |
50 | return true; |
51 | src_sg = sg_next(src_sg); |
52 | dst_sg = sg_next(dst_sg); |
53 | } |
54 | |
55 | return false; |
56 | } |
57 | |
58 | static int meson_cipher_do_fallback(struct skcipher_request *areq) |
59 | { |
60 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
61 | struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
62 | struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
63 | int err; |
64 | #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG |
65 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
66 | struct meson_alg_template *algt; |
67 | |
68 | algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); |
69 | algt->stat_fb++; |
70 | #endif |
71 | skcipher_request_set_tfm(req: &rctx->fallback_req, tfm: op->fallback_tfm); |
72 | skcipher_request_set_callback(req: &rctx->fallback_req, flags: areq->base.flags, |
73 | compl: areq->base.complete, data: areq->base.data); |
74 | skcipher_request_set_crypt(req: &rctx->fallback_req, src: areq->src, dst: areq->dst, |
75 | cryptlen: areq->cryptlen, iv: areq->iv); |
76 | |
77 | if (rctx->op_dir == MESON_DECRYPT) |
78 | err = crypto_skcipher_decrypt(req: &rctx->fallback_req); |
79 | else |
80 | err = crypto_skcipher_encrypt(req: &rctx->fallback_req); |
81 | return err; |
82 | } |
83 | |
84 | static int meson_cipher(struct skcipher_request *areq) |
85 | { |
86 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
87 | struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
88 | struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
89 | struct meson_dev *mc = op->mc; |
90 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm); |
91 | struct meson_alg_template *algt; |
92 | int flow = rctx->flow; |
93 | unsigned int todo, eat, len; |
94 | struct scatterlist *src_sg = areq->src; |
95 | struct scatterlist *dst_sg = areq->dst; |
96 | struct meson_desc *desc; |
97 | int nr_sgs, nr_sgd; |
98 | int i, err = 0; |
99 | unsigned int keyivlen, ivsize, offset, tloffset; |
100 | dma_addr_t phykeyiv; |
101 | void *backup_iv = NULL, *bkeyiv; |
102 | u32 v; |
103 | |
104 | algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); |
105 | |
106 | dev_dbg(mc->dev, "%s %s %u %x IV(%u) key=%u flow=%d\n" , __func__, |
107 | crypto_tfm_alg_name(areq->base.tfm), |
108 | areq->cryptlen, |
109 | rctx->op_dir, crypto_skcipher_ivsize(tfm), |
110 | op->keylen, flow); |
111 | |
112 | #ifdef CONFIG_CRYPTO_DEV_AMLOGIC_GXL_DEBUG |
113 | algt->stat_req++; |
114 | mc->chanlist[flow].stat_req++; |
115 | #endif |
116 | |
117 | /* |
118 | * The hardware expect a list of meson_desc structures. |
119 | * The 2 first structures store key |
120 | * The third stores IV |
121 | */ |
122 | bkeyiv = kzalloc(size: 48, GFP_KERNEL | GFP_DMA); |
123 | if (!bkeyiv) |
124 | return -ENOMEM; |
125 | |
126 | memcpy(bkeyiv, op->key, op->keylen); |
127 | keyivlen = op->keylen; |
128 | |
129 | ivsize = crypto_skcipher_ivsize(tfm); |
130 | if (areq->iv && ivsize > 0) { |
131 | if (ivsize > areq->cryptlen) { |
132 | dev_err(mc->dev, "invalid ivsize=%d vs len=%d\n" , ivsize, areq->cryptlen); |
133 | err = -EINVAL; |
134 | goto theend; |
135 | } |
136 | memcpy(bkeyiv + 32, areq->iv, ivsize); |
137 | keyivlen = 48; |
138 | if (rctx->op_dir == MESON_DECRYPT) { |
139 | backup_iv = kzalloc(size: ivsize, GFP_KERNEL); |
140 | if (!backup_iv) { |
141 | err = -ENOMEM; |
142 | goto theend; |
143 | } |
144 | offset = areq->cryptlen - ivsize; |
145 | scatterwalk_map_and_copy(buf: backup_iv, sg: areq->src, start: offset, |
146 | nbytes: ivsize, out: 0); |
147 | } |
148 | } |
149 | if (keyivlen == 24) |
150 | keyivlen = 32; |
151 | |
152 | phykeyiv = dma_map_single(mc->dev, bkeyiv, keyivlen, |
153 | DMA_TO_DEVICE); |
154 | err = dma_mapping_error(dev: mc->dev, dma_addr: phykeyiv); |
155 | if (err) { |
156 | dev_err(mc->dev, "Cannot DMA MAP KEY IV\n" ); |
157 | goto theend; |
158 | } |
159 | |
160 | tloffset = 0; |
161 | eat = 0; |
162 | i = 0; |
163 | while (keyivlen > eat) { |
164 | desc = &mc->chanlist[flow].tl[tloffset]; |
165 | memset(desc, 0, sizeof(struct meson_desc)); |
166 | todo = min(keyivlen - eat, 16u); |
167 | desc->t_src = cpu_to_le32(phykeyiv + i * 16); |
168 | desc->t_dst = cpu_to_le32(i * 16); |
169 | v = (MODE_KEY << 20) | DESC_OWN | 16; |
170 | desc->t_status = cpu_to_le32(v); |
171 | |
172 | eat += todo; |
173 | i++; |
174 | tloffset++; |
175 | } |
176 | |
177 | if (areq->src == areq->dst) { |
178 | nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), |
179 | DMA_BIDIRECTIONAL); |
180 | if (!nr_sgs) { |
181 | dev_err(mc->dev, "Invalid SG count %d\n" , nr_sgs); |
182 | err = -EINVAL; |
183 | goto theend; |
184 | } |
185 | nr_sgd = nr_sgs; |
186 | } else { |
187 | nr_sgs = dma_map_sg(mc->dev, areq->src, sg_nents(areq->src), |
188 | DMA_TO_DEVICE); |
189 | if (!nr_sgs || nr_sgs > MAXDESC - 3) { |
190 | dev_err(mc->dev, "Invalid SG count %d\n" , nr_sgs); |
191 | err = -EINVAL; |
192 | goto theend; |
193 | } |
194 | nr_sgd = dma_map_sg(mc->dev, areq->dst, sg_nents(areq->dst), |
195 | DMA_FROM_DEVICE); |
196 | if (!nr_sgd || nr_sgd > MAXDESC - 3) { |
197 | dev_err(mc->dev, "Invalid SG count %d\n" , nr_sgd); |
198 | err = -EINVAL; |
199 | goto theend; |
200 | } |
201 | } |
202 | |
203 | src_sg = areq->src; |
204 | dst_sg = areq->dst; |
205 | len = areq->cryptlen; |
206 | while (src_sg) { |
207 | desc = &mc->chanlist[flow].tl[tloffset]; |
208 | memset(desc, 0, sizeof(struct meson_desc)); |
209 | |
210 | desc->t_src = cpu_to_le32(sg_dma_address(src_sg)); |
211 | desc->t_dst = cpu_to_le32(sg_dma_address(dst_sg)); |
212 | todo = min(len, sg_dma_len(src_sg)); |
213 | v = (op->keymode << 20) | DESC_OWN | todo | (algt->blockmode << 26); |
214 | if (rctx->op_dir) |
215 | v |= DESC_ENCRYPTION; |
216 | len -= todo; |
217 | |
218 | if (!sg_next(src_sg)) |
219 | v |= DESC_LAST; |
220 | desc->t_status = cpu_to_le32(v); |
221 | tloffset++; |
222 | src_sg = sg_next(src_sg); |
223 | dst_sg = sg_next(dst_sg); |
224 | } |
225 | |
226 | reinit_completion(x: &mc->chanlist[flow].complete); |
227 | mc->chanlist[flow].status = 0; |
228 | writel(val: mc->chanlist[flow].t_phy | 2, addr: mc->base + (flow << 2)); |
229 | wait_for_completion_interruptible_timeout(x: &mc->chanlist[flow].complete, |
230 | timeout: msecs_to_jiffies(m: 500)); |
231 | if (mc->chanlist[flow].status == 0) { |
232 | dev_err(mc->dev, "DMA timeout for flow %d\n" , flow); |
233 | err = -EINVAL; |
234 | } |
235 | |
236 | dma_unmap_single(mc->dev, phykeyiv, keyivlen, DMA_TO_DEVICE); |
237 | |
238 | if (areq->src == areq->dst) { |
239 | dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_BIDIRECTIONAL); |
240 | } else { |
241 | dma_unmap_sg(mc->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); |
242 | dma_unmap_sg(mc->dev, areq->dst, sg_nents(areq->dst), DMA_FROM_DEVICE); |
243 | } |
244 | |
245 | if (areq->iv && ivsize > 0) { |
246 | if (rctx->op_dir == MESON_DECRYPT) { |
247 | memcpy(areq->iv, backup_iv, ivsize); |
248 | } else { |
249 | scatterwalk_map_and_copy(buf: areq->iv, sg: areq->dst, |
250 | start: areq->cryptlen - ivsize, |
251 | nbytes: ivsize, out: 0); |
252 | } |
253 | } |
254 | theend: |
255 | kfree_sensitive(objp: bkeyiv); |
256 | kfree_sensitive(objp: backup_iv); |
257 | |
258 | return err; |
259 | } |
260 | |
261 | int meson_handle_cipher_request(struct crypto_engine *engine, void *areq) |
262 | { |
263 | int err; |
264 | struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); |
265 | |
266 | err = meson_cipher(areq: breq); |
267 | local_bh_disable(); |
268 | crypto_finalize_skcipher_request(engine, req: breq, err); |
269 | local_bh_enable(); |
270 | |
271 | return 0; |
272 | } |
273 | |
274 | int meson_skdecrypt(struct skcipher_request *areq) |
275 | { |
276 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
277 | struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
278 | struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
279 | struct crypto_engine *engine; |
280 | int e; |
281 | |
282 | rctx->op_dir = MESON_DECRYPT; |
283 | if (meson_cipher_need_fallback(areq)) |
284 | return meson_cipher_do_fallback(areq); |
285 | e = get_engine_number(mc: op->mc); |
286 | engine = op->mc->chanlist[e].engine; |
287 | rctx->flow = e; |
288 | |
289 | return crypto_transfer_skcipher_request_to_engine(engine, req: areq); |
290 | } |
291 | |
292 | int meson_skencrypt(struct skcipher_request *areq) |
293 | { |
294 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req: areq); |
295 | struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
296 | struct meson_cipher_req_ctx *rctx = skcipher_request_ctx(req: areq); |
297 | struct crypto_engine *engine; |
298 | int e; |
299 | |
300 | rctx->op_dir = MESON_ENCRYPT; |
301 | if (meson_cipher_need_fallback(areq)) |
302 | return meson_cipher_do_fallback(areq); |
303 | e = get_engine_number(mc: op->mc); |
304 | engine = op->mc->chanlist[e].engine; |
305 | rctx->flow = e; |
306 | |
307 | return crypto_transfer_skcipher_request_to_engine(engine, req: areq); |
308 | } |
309 | |
310 | int meson_cipher_init(struct crypto_tfm *tfm) |
311 | { |
312 | struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); |
313 | struct meson_alg_template *algt; |
314 | const char *name = crypto_tfm_alg_name(tfm); |
315 | struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); |
316 | struct skcipher_alg *alg = crypto_skcipher_alg(tfm: sktfm); |
317 | |
318 | memset(op, 0, sizeof(struct meson_cipher_tfm_ctx)); |
319 | |
320 | algt = container_of(alg, struct meson_alg_template, alg.skcipher.base); |
321 | op->mc = algt->mc; |
322 | |
323 | op->fallback_tfm = crypto_alloc_skcipher(alg_name: name, type: 0, CRYPTO_ALG_NEED_FALLBACK); |
324 | if (IS_ERR(ptr: op->fallback_tfm)) { |
325 | dev_err(op->mc->dev, "ERROR: Cannot allocate fallback for %s %ld\n" , |
326 | name, PTR_ERR(op->fallback_tfm)); |
327 | return PTR_ERR(ptr: op->fallback_tfm); |
328 | } |
329 | |
330 | crypto_skcipher_set_reqsize(skcipher: sktfm, reqsize: sizeof(struct meson_cipher_req_ctx) + |
331 | crypto_skcipher_reqsize(tfm: op->fallback_tfm)); |
332 | |
333 | return 0; |
334 | } |
335 | |
336 | void meson_cipher_exit(struct crypto_tfm *tfm) |
337 | { |
338 | struct meson_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); |
339 | |
340 | kfree_sensitive(objp: op->key); |
341 | crypto_free_skcipher(tfm: op->fallback_tfm); |
342 | } |
343 | |
344 | int meson_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, |
345 | unsigned int keylen) |
346 | { |
347 | struct meson_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); |
348 | struct meson_dev *mc = op->mc; |
349 | |
350 | switch (keylen) { |
351 | case 128 / 8: |
352 | op->keymode = MODE_AES_128; |
353 | break; |
354 | case 192 / 8: |
355 | op->keymode = MODE_AES_192; |
356 | break; |
357 | case 256 / 8: |
358 | op->keymode = MODE_AES_256; |
359 | break; |
360 | default: |
361 | dev_dbg(mc->dev, "ERROR: Invalid keylen %u\n" , keylen); |
362 | return -EINVAL; |
363 | } |
364 | kfree_sensitive(objp: op->key); |
365 | op->keylen = keylen; |
366 | op->key = kmemdup(p: key, size: keylen, GFP_KERNEL | GFP_DMA); |
367 | if (!op->key) |
368 | return -ENOMEM; |
369 | |
370 | return crypto_skcipher_setkey(tfm: op->fallback_tfm, key, keylen); |
371 | } |
372 | |