1// SPDX-License-Identifier: GPL-2.0
2/*
3 * StarFive AES acceleration driver
4 *
5 * Copyright (c) 2022 StarFive Technology
6 */
7
8#include <crypto/engine.h>
9#include <crypto/gcm.h>
10#include <crypto/internal/aead.h>
11#include <crypto/internal/skcipher.h>
12#include <crypto/scatterwalk.h>
13#include "jh7110-cryp.h"
14#include <linux/err.h>
15#include <linux/iopoll.h>
16#include <linux/kernel.h>
17#include <linux/slab.h>
18#include <linux/string.h>
19
20#define STARFIVE_AES_REGS_OFFSET 0x100
21#define STARFIVE_AES_AESDIO0R (STARFIVE_AES_REGS_OFFSET + 0x0)
22#define STARFIVE_AES_KEY0 (STARFIVE_AES_REGS_OFFSET + 0x4)
23#define STARFIVE_AES_KEY1 (STARFIVE_AES_REGS_OFFSET + 0x8)
24#define STARFIVE_AES_KEY2 (STARFIVE_AES_REGS_OFFSET + 0xC)
25#define STARFIVE_AES_KEY3 (STARFIVE_AES_REGS_OFFSET + 0x10)
26#define STARFIVE_AES_KEY4 (STARFIVE_AES_REGS_OFFSET + 0x14)
27#define STARFIVE_AES_KEY5 (STARFIVE_AES_REGS_OFFSET + 0x18)
28#define STARFIVE_AES_KEY6 (STARFIVE_AES_REGS_OFFSET + 0x1C)
29#define STARFIVE_AES_KEY7 (STARFIVE_AES_REGS_OFFSET + 0x20)
30#define STARFIVE_AES_CSR (STARFIVE_AES_REGS_OFFSET + 0x24)
31#define STARFIVE_AES_IV0 (STARFIVE_AES_REGS_OFFSET + 0x28)
32#define STARFIVE_AES_IV1 (STARFIVE_AES_REGS_OFFSET + 0x2C)
33#define STARFIVE_AES_IV2 (STARFIVE_AES_REGS_OFFSET + 0x30)
34#define STARFIVE_AES_IV3 (STARFIVE_AES_REGS_OFFSET + 0x34)
35#define STARFIVE_AES_NONCE0 (STARFIVE_AES_REGS_OFFSET + 0x3C)
36#define STARFIVE_AES_NONCE1 (STARFIVE_AES_REGS_OFFSET + 0x40)
37#define STARFIVE_AES_NONCE2 (STARFIVE_AES_REGS_OFFSET + 0x44)
38#define STARFIVE_AES_NONCE3 (STARFIVE_AES_REGS_OFFSET + 0x48)
39#define STARFIVE_AES_ALEN0 (STARFIVE_AES_REGS_OFFSET + 0x4C)
40#define STARFIVE_AES_ALEN1 (STARFIVE_AES_REGS_OFFSET + 0x50)
41#define STARFIVE_AES_MLEN0 (STARFIVE_AES_REGS_OFFSET + 0x54)
42#define STARFIVE_AES_MLEN1 (STARFIVE_AES_REGS_OFFSET + 0x58)
43#define STARFIVE_AES_IVLEN (STARFIVE_AES_REGS_OFFSET + 0x5C)
44
45#define FLG_MODE_MASK GENMASK(2, 0)
46#define FLG_ENCRYPT BIT(4)
47
48/* Misc */
49#define CCM_B0_ADATA 0x40
50#define AES_BLOCK_32 (AES_BLOCK_SIZE / sizeof(u32))
51
52static inline int starfive_aes_wait_busy(struct starfive_cryp_dev *cryp)
53{
54 u32 status;
55
56 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
57 !(status & STARFIVE_AES_BUSY), 10, 100000);
58}
59
60static inline int starfive_aes_wait_keydone(struct starfive_cryp_dev *cryp)
61{
62 u32 status;
63
64 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
65 (status & STARFIVE_AES_KEY_DONE), 10, 100000);
66}
67
68static inline int starfive_aes_wait_gcmdone(struct starfive_cryp_dev *cryp)
69{
70 u32 status;
71
72 return readl_relaxed_poll_timeout(cryp->base + STARFIVE_AES_CSR, status,
73 (status & STARFIVE_AES_GCM_DONE), 10, 100000);
74}
75
76static inline int is_gcm(struct starfive_cryp_dev *cryp)
77{
78 return (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_GCM;
79}
80
81static inline int is_encrypt(struct starfive_cryp_dev *cryp)
82{
83 return cryp->flags & FLG_ENCRYPT;
84}
85
86static void starfive_aes_aead_hw_start(struct starfive_cryp_ctx *ctx, u32 hw_mode)
87{
88 struct starfive_cryp_dev *cryp = ctx->cryp;
89 unsigned int value;
90
91 switch (hw_mode) {
92 case STARFIVE_AES_MODE_GCM:
93 value = readl(addr: ctx->cryp->base + STARFIVE_AES_CSR);
94 value |= STARFIVE_AES_GCM_START;
95 writel(val: value, addr: cryp->base + STARFIVE_AES_CSR);
96 starfive_aes_wait_gcmdone(cryp);
97 break;
98 case STARFIVE_AES_MODE_CCM:
99 value = readl(addr: ctx->cryp->base + STARFIVE_AES_CSR);
100 value |= STARFIVE_AES_CCM_START;
101 writel(val: value, addr: cryp->base + STARFIVE_AES_CSR);
102 break;
103 }
104}
105
106static inline void starfive_aes_set_ivlen(struct starfive_cryp_ctx *ctx)
107{
108 struct starfive_cryp_dev *cryp = ctx->cryp;
109
110 if (is_gcm(cryp))
111 writel(GCM_AES_IV_SIZE, addr: cryp->base + STARFIVE_AES_IVLEN);
112 else
113 writel(AES_BLOCK_SIZE, addr: cryp->base + STARFIVE_AES_IVLEN);
114}
115
116static inline void starfive_aes_set_alen(struct starfive_cryp_ctx *ctx)
117{
118 struct starfive_cryp_dev *cryp = ctx->cryp;
119
120 writel(upper_32_bits(cryp->assoclen), addr: cryp->base + STARFIVE_AES_ALEN0);
121 writel(lower_32_bits(cryp->assoclen), addr: cryp->base + STARFIVE_AES_ALEN1);
122}
123
124static inline void starfive_aes_set_mlen(struct starfive_cryp_ctx *ctx)
125{
126 struct starfive_cryp_dev *cryp = ctx->cryp;
127
128 writel(upper_32_bits(cryp->total_in), addr: cryp->base + STARFIVE_AES_MLEN0);
129 writel(lower_32_bits(cryp->total_in), addr: cryp->base + STARFIVE_AES_MLEN1);
130}
131
132static inline int starfive_aes_ccm_check_iv(const u8 *iv)
133{
134 /* 2 <= L <= 8, so 1 <= L' <= 7. */
135 if (iv[0] < 1 || iv[0] > 7)
136 return -EINVAL;
137
138 return 0;
139}
140
141static int starfive_aes_write_iv(struct starfive_cryp_ctx *ctx, u32 *iv)
142{
143 struct starfive_cryp_dev *cryp = ctx->cryp;
144
145 writel(val: iv[0], addr: cryp->base + STARFIVE_AES_IV0);
146 writel(val: iv[1], addr: cryp->base + STARFIVE_AES_IV1);
147 writel(val: iv[2], addr: cryp->base + STARFIVE_AES_IV2);
148
149 if (is_gcm(cryp)) {
150 if (starfive_aes_wait_gcmdone(cryp))
151 return -ETIMEDOUT;
152
153 return 0;
154 }
155
156 writel(val: iv[3], addr: cryp->base + STARFIVE_AES_IV3);
157
158 return 0;
159}
160
161static inline void starfive_aes_get_iv(struct starfive_cryp_dev *cryp, u32 *iv)
162{
163 iv[0] = readl(addr: cryp->base + STARFIVE_AES_IV0);
164 iv[1] = readl(addr: cryp->base + STARFIVE_AES_IV1);
165 iv[2] = readl(addr: cryp->base + STARFIVE_AES_IV2);
166 iv[3] = readl(addr: cryp->base + STARFIVE_AES_IV3);
167}
168
169static inline void starfive_aes_write_nonce(struct starfive_cryp_ctx *ctx, u32 *nonce)
170{
171 struct starfive_cryp_dev *cryp = ctx->cryp;
172
173 writel(val: nonce[0], addr: cryp->base + STARFIVE_AES_NONCE0);
174 writel(val: nonce[1], addr: cryp->base + STARFIVE_AES_NONCE1);
175 writel(val: nonce[2], addr: cryp->base + STARFIVE_AES_NONCE2);
176 writel(val: nonce[3], addr: cryp->base + STARFIVE_AES_NONCE3);
177}
178
179static int starfive_aes_write_key(struct starfive_cryp_ctx *ctx)
180{
181 struct starfive_cryp_dev *cryp = ctx->cryp;
182 u32 *key = (u32 *)ctx->key;
183
184 if (ctx->keylen >= AES_KEYSIZE_128) {
185 writel(val: key[0], addr: cryp->base + STARFIVE_AES_KEY0);
186 writel(val: key[1], addr: cryp->base + STARFIVE_AES_KEY1);
187 writel(val: key[2], addr: cryp->base + STARFIVE_AES_KEY2);
188 writel(val: key[3], addr: cryp->base + STARFIVE_AES_KEY3);
189 }
190
191 if (ctx->keylen >= AES_KEYSIZE_192) {
192 writel(val: key[4], addr: cryp->base + STARFIVE_AES_KEY4);
193 writel(val: key[5], addr: cryp->base + STARFIVE_AES_KEY5);
194 }
195
196 if (ctx->keylen >= AES_KEYSIZE_256) {
197 writel(val: key[6], addr: cryp->base + STARFIVE_AES_KEY6);
198 writel(val: key[7], addr: cryp->base + STARFIVE_AES_KEY7);
199 }
200
201 if (starfive_aes_wait_keydone(cryp))
202 return -ETIMEDOUT;
203
204 return 0;
205}
206
207static int starfive_aes_ccm_init(struct starfive_cryp_ctx *ctx)
208{
209 struct starfive_cryp_dev *cryp = ctx->cryp;
210 u8 iv[AES_BLOCK_SIZE], b0[AES_BLOCK_SIZE];
211 unsigned int textlen;
212
213 memcpy(iv, cryp->req.areq->iv, AES_BLOCK_SIZE);
214 memset(iv + AES_BLOCK_SIZE - 1 - iv[0], 0, iv[0] + 1);
215
216 /* Build B0 */
217 memcpy(b0, iv, AES_BLOCK_SIZE);
218
219 b0[0] |= (8 * ((cryp->authsize - 2) / 2));
220
221 if (cryp->assoclen)
222 b0[0] |= CCM_B0_ADATA;
223
224 textlen = cryp->total_in;
225
226 b0[AES_BLOCK_SIZE - 2] = textlen >> 8;
227 b0[AES_BLOCK_SIZE - 1] = textlen & 0xFF;
228
229 starfive_aes_write_nonce(ctx, nonce: (u32 *)b0);
230
231 return 0;
232}
233
234static int starfive_aes_hw_init(struct starfive_cryp_ctx *ctx)
235{
236 struct starfive_cryp_request_ctx *rctx = ctx->rctx;
237 struct starfive_cryp_dev *cryp = ctx->cryp;
238 u32 hw_mode;
239
240 /* reset */
241 rctx->csr.aes.v = 0;
242 rctx->csr.aes.aesrst = 1;
243 writel(val: rctx->csr.aes.v, addr: cryp->base + STARFIVE_AES_CSR);
244
245 /* csr setup */
246 hw_mode = cryp->flags & FLG_MODE_MASK;
247
248 rctx->csr.aes.v = 0;
249
250 switch (ctx->keylen) {
251 case AES_KEYSIZE_128:
252 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_128;
253 break;
254 case AES_KEYSIZE_192:
255 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_192;
256 break;
257 case AES_KEYSIZE_256:
258 rctx->csr.aes.keymode = STARFIVE_AES_KEYMODE_256;
259 break;
260 }
261
262 rctx->csr.aes.mode = hw_mode;
263 rctx->csr.aes.cmode = !is_encrypt(cryp);
264 rctx->csr.aes.ie = 1;
265 rctx->csr.aes.stmode = STARFIVE_AES_MODE_XFB_1;
266
267 if (cryp->side_chan) {
268 rctx->csr.aes.delay_aes = 1;
269 rctx->csr.aes.vaes_start = 1;
270 }
271
272 writel(val: rctx->csr.aes.v, addr: cryp->base + STARFIVE_AES_CSR);
273
274 cryp->err = starfive_aes_write_key(ctx);
275 if (cryp->err)
276 return cryp->err;
277
278 switch (hw_mode) {
279 case STARFIVE_AES_MODE_GCM:
280 starfive_aes_set_alen(ctx);
281 starfive_aes_set_mlen(ctx);
282 starfive_aes_set_ivlen(ctx);
283 starfive_aes_aead_hw_start(ctx, hw_mode);
284 starfive_aes_write_iv(ctx, iv: (void *)cryp->req.areq->iv);
285 break;
286 case STARFIVE_AES_MODE_CCM:
287 starfive_aes_set_alen(ctx);
288 starfive_aes_set_mlen(ctx);
289 starfive_aes_ccm_init(ctx);
290 starfive_aes_aead_hw_start(ctx, hw_mode);
291 break;
292 case STARFIVE_AES_MODE_CBC:
293 case STARFIVE_AES_MODE_CTR:
294 starfive_aes_write_iv(ctx, iv: (void *)cryp->req.sreq->iv);
295 break;
296 default:
297 break;
298 }
299
300 return cryp->err;
301}
302
303static int starfive_aes_read_authtag(struct starfive_cryp_dev *cryp)
304{
305 int i, start_addr;
306
307 if (starfive_aes_wait_busy(cryp))
308 return dev_err_probe(dev: cryp->dev, err: -ETIMEDOUT,
309 fmt: "Timeout waiting for tag generation.");
310
311 start_addr = STARFIVE_AES_NONCE0;
312
313 if (is_gcm(cryp))
314 for (i = 0; i < AES_BLOCK_32; i++, start_addr += 4)
315 cryp->tag_out[i] = readl(addr: cryp->base + start_addr);
316 else
317 for (i = 0; i < AES_BLOCK_32; i++)
318 cryp->tag_out[i] = readl(addr: cryp->base + STARFIVE_AES_AESDIO0R);
319
320 if (is_encrypt(cryp)) {
321 scatterwalk_copychunks(buf: cryp->tag_out, walk: &cryp->out_walk, nbytes: cryp->authsize, out: 1);
322 } else {
323 scatterwalk_copychunks(buf: cryp->tag_in, walk: &cryp->in_walk, nbytes: cryp->authsize, out: 0);
324
325 if (crypto_memneq(a: cryp->tag_in, b: cryp->tag_out, size: cryp->authsize))
326 return dev_err_probe(dev: cryp->dev, err: -EBADMSG, fmt: "Failed tag verification\n");
327 }
328
329 return 0;
330}
331
332static void starfive_aes_finish_req(struct starfive_cryp_dev *cryp)
333{
334 union starfive_aes_csr csr;
335 int err = cryp->err;
336
337 if (!err && cryp->authsize)
338 err = starfive_aes_read_authtag(cryp);
339
340 if (!err && ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC ||
341 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CTR))
342 starfive_aes_get_iv(cryp, iv: (void *)cryp->req.sreq->iv);
343
344 /* reset irq flags*/
345 csr.v = 0;
346 csr.aesrst = 1;
347 writel(val: csr.v, addr: cryp->base + STARFIVE_AES_CSR);
348
349 if (cryp->authsize)
350 crypto_finalize_aead_request(engine: cryp->engine, req: cryp->req.areq, err);
351 else
352 crypto_finalize_skcipher_request(engine: cryp->engine, req: cryp->req.sreq,
353 err);
354}
355
356void starfive_aes_done_task(unsigned long param)
357{
358 struct starfive_cryp_dev *cryp = (struct starfive_cryp_dev *)param;
359 u32 block[AES_BLOCK_32];
360 u32 stat;
361 int i;
362
363 for (i = 0; i < AES_BLOCK_32; i++)
364 block[i] = readl(addr: cryp->base + STARFIVE_AES_AESDIO0R);
365
366 scatterwalk_copychunks(buf: block, walk: &cryp->out_walk, min_t(size_t, AES_BLOCK_SIZE,
367 cryp->total_out), out: 1);
368
369 cryp->total_out -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_out);
370
371 if (!cryp->total_out) {
372 starfive_aes_finish_req(cryp);
373 return;
374 }
375
376 memset(block, 0, AES_BLOCK_SIZE);
377 scatterwalk_copychunks(buf: block, walk: &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
378 cryp->total_in), out: 0);
379 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
380
381 for (i = 0; i < AES_BLOCK_32; i++)
382 writel(val: block[i], addr: cryp->base + STARFIVE_AES_AESDIO0R);
383
384 stat = readl(addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
385 stat &= ~STARFIVE_IE_MASK_AES_DONE;
386 writel(val: stat, addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
387}
388
389static int starfive_aes_gcm_write_adata(struct starfive_cryp_ctx *ctx)
390{
391 struct starfive_cryp_dev *cryp = ctx->cryp;
392 struct starfive_cryp_request_ctx *rctx = ctx->rctx;
393 u32 *buffer;
394 int total_len, loop;
395
396 total_len = ALIGN(cryp->assoclen, AES_BLOCK_SIZE) / sizeof(unsigned int);
397 buffer = (u32 *)rctx->adata;
398
399 for (loop = 0; loop < total_len; loop += 4) {
400 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_NONCE0);
401 buffer++;
402 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_NONCE1);
403 buffer++;
404 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_NONCE2);
405 buffer++;
406 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_NONCE3);
407 buffer++;
408 }
409
410 if (starfive_aes_wait_gcmdone(cryp))
411 return dev_err_probe(dev: cryp->dev, err: -ETIMEDOUT,
412 fmt: "Timeout processing gcm aad block");
413
414 return 0;
415}
416
417static int starfive_aes_ccm_write_adata(struct starfive_cryp_ctx *ctx)
418{
419 struct starfive_cryp_dev *cryp = ctx->cryp;
420 struct starfive_cryp_request_ctx *rctx = ctx->rctx;
421 u32 *buffer;
422 u8 *ci;
423 int total_len, loop;
424
425 total_len = cryp->assoclen;
426
427 ci = rctx->adata;
428 writeb(val: *ci, addr: cryp->base + STARFIVE_AES_AESDIO0R);
429 ci++;
430 writeb(val: *ci, addr: cryp->base + STARFIVE_AES_AESDIO0R);
431 ci++;
432 total_len -= 2;
433 buffer = (u32 *)ci;
434
435 for (loop = 0; loop < 3; loop++, buffer++)
436 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_AESDIO0R);
437
438 total_len -= 12;
439
440 while (total_len > 0) {
441 for (loop = 0; loop < AES_BLOCK_32; loop++, buffer++)
442 writel(val: *buffer, addr: cryp->base + STARFIVE_AES_AESDIO0R);
443
444 total_len -= AES_BLOCK_SIZE;
445 }
446
447 if (starfive_aes_wait_busy(cryp))
448 return dev_err_probe(dev: cryp->dev, err: -ETIMEDOUT,
449 fmt: "Timeout processing ccm aad block");
450
451 return 0;
452}
453
454static int starfive_aes_prepare_req(struct skcipher_request *req,
455 struct aead_request *areq)
456{
457 struct starfive_cryp_ctx *ctx;
458 struct starfive_cryp_request_ctx *rctx;
459 struct starfive_cryp_dev *cryp;
460
461 if (!req && !areq)
462 return -EINVAL;
463
464 ctx = req ? crypto_skcipher_ctx(tfm: crypto_skcipher_reqtfm(req)) :
465 crypto_aead_ctx(tfm: crypto_aead_reqtfm(req: areq));
466
467 cryp = ctx->cryp;
468 rctx = req ? skcipher_request_ctx(req) : aead_request_ctx(req: areq);
469
470 if (req) {
471 cryp->req.sreq = req;
472 cryp->total_in = req->cryptlen;
473 cryp->total_out = req->cryptlen;
474 cryp->assoclen = 0;
475 cryp->authsize = 0;
476 } else {
477 cryp->req.areq = areq;
478 cryp->assoclen = areq->assoclen;
479 cryp->authsize = crypto_aead_authsize(tfm: crypto_aead_reqtfm(req: areq));
480 if (is_encrypt(cryp)) {
481 cryp->total_in = areq->cryptlen;
482 cryp->total_out = areq->cryptlen;
483 } else {
484 cryp->total_in = areq->cryptlen - cryp->authsize;
485 cryp->total_out = cryp->total_in;
486 }
487 }
488
489 rctx->in_sg = req ? req->src : areq->src;
490 scatterwalk_start(walk: &cryp->in_walk, sg: rctx->in_sg);
491
492 rctx->out_sg = req ? req->dst : areq->dst;
493 scatterwalk_start(walk: &cryp->out_walk, sg: rctx->out_sg);
494
495 if (cryp->assoclen) {
496 rctx->adata = kzalloc(size: cryp->assoclen + AES_BLOCK_SIZE, GFP_KERNEL);
497 if (!rctx->adata)
498 return dev_err_probe(dev: cryp->dev, err: -ENOMEM,
499 fmt: "Failed to alloc memory for adata");
500
501 scatterwalk_copychunks(buf: rctx->adata, walk: &cryp->in_walk, nbytes: cryp->assoclen, out: 0);
502 scatterwalk_copychunks(NULL, walk: &cryp->out_walk, nbytes: cryp->assoclen, out: 2);
503 }
504
505 ctx->rctx = rctx;
506
507 return starfive_aes_hw_init(ctx);
508}
509
510static int starfive_aes_do_one_req(struct crypto_engine *engine, void *areq)
511{
512 struct skcipher_request *req =
513 container_of(areq, struct skcipher_request, base);
514 struct starfive_cryp_ctx *ctx =
515 crypto_skcipher_ctx(tfm: crypto_skcipher_reqtfm(req));
516 struct starfive_cryp_dev *cryp = ctx->cryp;
517 u32 block[AES_BLOCK_32];
518 u32 stat;
519 int err;
520 int i;
521
522 err = starfive_aes_prepare_req(req, NULL);
523 if (err)
524 return err;
525
526 /*
527 * Write first plain/ciphertext block to start the module
528 * then let irq tasklet handle the rest of the data blocks.
529 */
530 scatterwalk_copychunks(buf: block, walk: &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
531 cryp->total_in), out: 0);
532 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
533
534 for (i = 0; i < AES_BLOCK_32; i++)
535 writel(val: block[i], addr: cryp->base + STARFIVE_AES_AESDIO0R);
536
537 stat = readl(addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
538 stat &= ~STARFIVE_IE_MASK_AES_DONE;
539 writel(val: stat, addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
540
541 return 0;
542}
543
544static int starfive_aes_init_tfm(struct crypto_skcipher *tfm)
545{
546 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
547
548 ctx->cryp = starfive_cryp_find_dev(ctx);
549 if (!ctx->cryp)
550 return -ENODEV;
551
552 crypto_skcipher_set_reqsize(skcipher: tfm, reqsize: sizeof(struct starfive_cryp_request_ctx) +
553 sizeof(struct skcipher_request));
554
555 return 0;
556}
557
558static int starfive_aes_aead_do_one_req(struct crypto_engine *engine, void *areq)
559{
560 struct aead_request *req =
561 container_of(areq, struct aead_request, base);
562 struct starfive_cryp_ctx *ctx =
563 crypto_aead_ctx(tfm: crypto_aead_reqtfm(req));
564 struct starfive_cryp_dev *cryp = ctx->cryp;
565 struct starfive_cryp_request_ctx *rctx;
566 u32 block[AES_BLOCK_32];
567 u32 stat;
568 int err;
569 int i;
570
571 err = starfive_aes_prepare_req(NULL, areq: req);
572 if (err)
573 return err;
574
575 rctx = ctx->rctx;
576
577 if (!cryp->assoclen)
578 goto write_text;
579
580 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CCM)
581 cryp->err = starfive_aes_ccm_write_adata(ctx);
582 else
583 cryp->err = starfive_aes_gcm_write_adata(ctx);
584
585 kfree(objp: rctx->adata);
586
587 if (cryp->err)
588 return cryp->err;
589
590write_text:
591 if (!cryp->total_in)
592 goto finish_req;
593
594 /*
595 * Write first plain/ciphertext block to start the module
596 * then let irq tasklet handle the rest of the data blocks.
597 */
598 scatterwalk_copychunks(buf: block, walk: &cryp->in_walk, min_t(size_t, AES_BLOCK_SIZE,
599 cryp->total_in), out: 0);
600 cryp->total_in -= min_t(size_t, AES_BLOCK_SIZE, cryp->total_in);
601
602 for (i = 0; i < AES_BLOCK_32; i++)
603 writel(val: block[i], addr: cryp->base + STARFIVE_AES_AESDIO0R);
604
605 stat = readl(addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
606 stat &= ~STARFIVE_IE_MASK_AES_DONE;
607 writel(val: stat, addr: cryp->base + STARFIVE_IE_MASK_OFFSET);
608
609 return 0;
610
611finish_req:
612 starfive_aes_finish_req(cryp);
613 return 0;
614}
615
616static int starfive_aes_aead_init_tfm(struct crypto_aead *tfm)
617{
618 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
619 struct starfive_cryp_dev *cryp = ctx->cryp;
620 struct crypto_tfm *aead = crypto_aead_tfm(tfm);
621 struct crypto_alg *alg = aead->__crt_alg;
622
623 ctx->cryp = starfive_cryp_find_dev(ctx);
624 if (!ctx->cryp)
625 return -ENODEV;
626
627 if (alg->cra_flags & CRYPTO_ALG_NEED_FALLBACK) {
628 ctx->aead_fbk = crypto_alloc_aead(alg_name: alg->cra_name, type: 0,
629 CRYPTO_ALG_NEED_FALLBACK);
630 if (IS_ERR(ptr: ctx->aead_fbk))
631 return dev_err_probe(dev: cryp->dev, err: PTR_ERR(ptr: ctx->aead_fbk),
632 fmt: "%s() failed to allocate fallback for %s\n",
633 __func__, alg->cra_name);
634 }
635
636 crypto_aead_set_reqsize(aead: tfm, reqsize: sizeof(struct starfive_cryp_ctx) +
637 sizeof(struct aead_request));
638
639 return 0;
640}
641
642static void starfive_aes_aead_exit_tfm(struct crypto_aead *tfm)
643{
644 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
645
646 crypto_free_aead(tfm: ctx->aead_fbk);
647}
648
649static int starfive_aes_crypt(struct skcipher_request *req, unsigned long flags)
650{
651 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
652 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
653 struct starfive_cryp_dev *cryp = ctx->cryp;
654 unsigned int blocksize_align = crypto_skcipher_blocksize(tfm) - 1;
655
656 cryp->flags = flags;
657
658 if ((cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_ECB ||
659 (cryp->flags & FLG_MODE_MASK) == STARFIVE_AES_MODE_CBC)
660 if (req->cryptlen & blocksize_align)
661 return -EINVAL;
662
663 return crypto_transfer_skcipher_request_to_engine(engine: cryp->engine, req);
664}
665
666static int starfive_aes_aead_crypt(struct aead_request *req, unsigned long flags)
667{
668 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm: crypto_aead_reqtfm(req));
669 struct starfive_cryp_dev *cryp = ctx->cryp;
670
671 cryp->flags = flags;
672
673 /*
674 * HW engine could not perform CCM tag verification on
675 * non-blocksize aligned text, use fallback algo instead
676 */
677 if (ctx->aead_fbk && !is_encrypt(cryp)) {
678 struct aead_request *subreq = aead_request_ctx(req);
679
680 aead_request_set_tfm(req: subreq, tfm: ctx->aead_fbk);
681 aead_request_set_callback(req: subreq, flags: req->base.flags,
682 compl: req->base.complete, data: req->base.data);
683 aead_request_set_crypt(req: subreq, src: req->src,
684 dst: req->dst, cryptlen: req->cryptlen, iv: req->iv);
685 aead_request_set_ad(req: subreq, assoclen: req->assoclen);
686
687 return crypto_aead_decrypt(req: subreq);
688 }
689
690 return crypto_transfer_aead_request_to_engine(engine: cryp->engine, req);
691}
692
693static int starfive_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
694 unsigned int keylen)
695{
696 struct starfive_cryp_ctx *ctx = crypto_skcipher_ctx(tfm);
697
698 if (!key || !keylen)
699 return -EINVAL;
700
701 if (keylen != AES_KEYSIZE_128 &&
702 keylen != AES_KEYSIZE_192 &&
703 keylen != AES_KEYSIZE_256)
704 return -EINVAL;
705
706 memcpy(ctx->key, key, keylen);
707 ctx->keylen = keylen;
708
709 return 0;
710}
711
712static int starfive_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key,
713 unsigned int keylen)
714{
715 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
716
717 if (!key || !keylen)
718 return -EINVAL;
719
720 if (keylen != AES_KEYSIZE_128 &&
721 keylen != AES_KEYSIZE_192 &&
722 keylen != AES_KEYSIZE_256)
723 return -EINVAL;
724
725 memcpy(ctx->key, key, keylen);
726 ctx->keylen = keylen;
727
728 if (ctx->aead_fbk)
729 return crypto_aead_setkey(tfm: ctx->aead_fbk, key, keylen);
730
731 return 0;
732}
733
734static int starfive_aes_gcm_setauthsize(struct crypto_aead *tfm,
735 unsigned int authsize)
736{
737 return crypto_gcm_check_authsize(authsize);
738}
739
740static int starfive_aes_ccm_setauthsize(struct crypto_aead *tfm,
741 unsigned int authsize)
742{
743 struct starfive_cryp_ctx *ctx = crypto_aead_ctx(tfm);
744
745 switch (authsize) {
746 case 4:
747 case 6:
748 case 8:
749 case 10:
750 case 12:
751 case 14:
752 case 16:
753 break;
754 default:
755 return -EINVAL;
756 }
757
758 return crypto_aead_setauthsize(tfm: ctx->aead_fbk, authsize);
759}
760
761static int starfive_aes_ecb_encrypt(struct skcipher_request *req)
762{
763 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB | FLG_ENCRYPT);
764}
765
766static int starfive_aes_ecb_decrypt(struct skcipher_request *req)
767{
768 return starfive_aes_crypt(req, STARFIVE_AES_MODE_ECB);
769}
770
771static int starfive_aes_cbc_encrypt(struct skcipher_request *req)
772{
773 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC | FLG_ENCRYPT);
774}
775
776static int starfive_aes_cbc_decrypt(struct skcipher_request *req)
777{
778 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CBC);
779}
780
781static int starfive_aes_ctr_encrypt(struct skcipher_request *req)
782{
783 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR | FLG_ENCRYPT);
784}
785
786static int starfive_aes_ctr_decrypt(struct skcipher_request *req)
787{
788 return starfive_aes_crypt(req, STARFIVE_AES_MODE_CTR);
789}
790
791static int starfive_aes_gcm_encrypt(struct aead_request *req)
792{
793 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM | FLG_ENCRYPT);
794}
795
796static int starfive_aes_gcm_decrypt(struct aead_request *req)
797{
798 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_GCM);
799}
800
801static int starfive_aes_ccm_encrypt(struct aead_request *req)
802{
803 int ret;
804
805 ret = starfive_aes_ccm_check_iv(iv: req->iv);
806 if (ret)
807 return ret;
808
809 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM | FLG_ENCRYPT);
810}
811
812static int starfive_aes_ccm_decrypt(struct aead_request *req)
813{
814 int ret;
815
816 ret = starfive_aes_ccm_check_iv(iv: req->iv);
817 if (ret)
818 return ret;
819
820 return starfive_aes_aead_crypt(req, STARFIVE_AES_MODE_CCM);
821}
822
823static struct skcipher_engine_alg skcipher_algs[] = {
824{
825 .base.init = starfive_aes_init_tfm,
826 .base.setkey = starfive_aes_setkey,
827 .base.encrypt = starfive_aes_ecb_encrypt,
828 .base.decrypt = starfive_aes_ecb_decrypt,
829 .base.min_keysize = AES_MIN_KEY_SIZE,
830 .base.max_keysize = AES_MAX_KEY_SIZE,
831 .base.base = {
832 .cra_name = "ecb(aes)",
833 .cra_driver_name = "starfive-ecb-aes",
834 .cra_priority = 200,
835 .cra_flags = CRYPTO_ALG_ASYNC,
836 .cra_blocksize = AES_BLOCK_SIZE,
837 .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
838 .cra_alignmask = 0xf,
839 .cra_module = THIS_MODULE,
840 },
841 .op = {
842 .do_one_request = starfive_aes_do_one_req,
843 },
844}, {
845 .base.init = starfive_aes_init_tfm,
846 .base.setkey = starfive_aes_setkey,
847 .base.encrypt = starfive_aes_cbc_encrypt,
848 .base.decrypt = starfive_aes_cbc_decrypt,
849 .base.min_keysize = AES_MIN_KEY_SIZE,
850 .base.max_keysize = AES_MAX_KEY_SIZE,
851 .base.ivsize = AES_BLOCK_SIZE,
852 .base.base = {
853 .cra_name = "cbc(aes)",
854 .cra_driver_name = "starfive-cbc-aes",
855 .cra_priority = 200,
856 .cra_flags = CRYPTO_ALG_ASYNC,
857 .cra_blocksize = AES_BLOCK_SIZE,
858 .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
859 .cra_alignmask = 0xf,
860 .cra_module = THIS_MODULE,
861 },
862 .op = {
863 .do_one_request = starfive_aes_do_one_req,
864 },
865}, {
866 .base.init = starfive_aes_init_tfm,
867 .base.setkey = starfive_aes_setkey,
868 .base.encrypt = starfive_aes_ctr_encrypt,
869 .base.decrypt = starfive_aes_ctr_decrypt,
870 .base.min_keysize = AES_MIN_KEY_SIZE,
871 .base.max_keysize = AES_MAX_KEY_SIZE,
872 .base.ivsize = AES_BLOCK_SIZE,
873 .base.base = {
874 .cra_name = "ctr(aes)",
875 .cra_driver_name = "starfive-ctr-aes",
876 .cra_priority = 200,
877 .cra_flags = CRYPTO_ALG_ASYNC,
878 .cra_blocksize = 1,
879 .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
880 .cra_alignmask = 0xf,
881 .cra_module = THIS_MODULE,
882 },
883 .op = {
884 .do_one_request = starfive_aes_do_one_req,
885 },
886},
887};
888
889static struct aead_engine_alg aead_algs[] = {
890{
891 .base.setkey = starfive_aes_aead_setkey,
892 .base.setauthsize = starfive_aes_gcm_setauthsize,
893 .base.encrypt = starfive_aes_gcm_encrypt,
894 .base.decrypt = starfive_aes_gcm_decrypt,
895 .base.init = starfive_aes_aead_init_tfm,
896 .base.exit = starfive_aes_aead_exit_tfm,
897 .base.ivsize = GCM_AES_IV_SIZE,
898 .base.maxauthsize = AES_BLOCK_SIZE,
899 .base.base = {
900 .cra_name = "gcm(aes)",
901 .cra_driver_name = "starfive-gcm-aes",
902 .cra_priority = 200,
903 .cra_flags = CRYPTO_ALG_ASYNC,
904 .cra_blocksize = 1,
905 .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
906 .cra_alignmask = 0xf,
907 .cra_module = THIS_MODULE,
908 },
909 .op = {
910 .do_one_request = starfive_aes_aead_do_one_req,
911 },
912}, {
913 .base.setkey = starfive_aes_aead_setkey,
914 .base.setauthsize = starfive_aes_ccm_setauthsize,
915 .base.encrypt = starfive_aes_ccm_encrypt,
916 .base.decrypt = starfive_aes_ccm_decrypt,
917 .base.init = starfive_aes_aead_init_tfm,
918 .base.exit = starfive_aes_aead_exit_tfm,
919 .base.ivsize = AES_BLOCK_SIZE,
920 .base.maxauthsize = AES_BLOCK_SIZE,
921 .base.base = {
922 .cra_name = "ccm(aes)",
923 .cra_driver_name = "starfive-ccm-aes",
924 .cra_priority = 200,
925 .cra_flags = CRYPTO_ALG_ASYNC |
926 CRYPTO_ALG_NEED_FALLBACK,
927 .cra_blocksize = 1,
928 .cra_ctxsize = sizeof(struct starfive_cryp_ctx),
929 .cra_alignmask = 0xf,
930 .cra_module = THIS_MODULE,
931 },
932 .op = {
933 .do_one_request = starfive_aes_aead_do_one_req,
934 },
935},
936};
937
938int starfive_aes_register_algs(void)
939{
940 int ret;
941
942 ret = crypto_engine_register_skciphers(algs: skcipher_algs, ARRAY_SIZE(skcipher_algs));
943 if (ret)
944 return ret;
945
946 ret = crypto_engine_register_aeads(algs: aead_algs, ARRAY_SIZE(aead_algs));
947 if (ret)
948 crypto_engine_unregister_skciphers(algs: skcipher_algs, ARRAY_SIZE(skcipher_algs));
949
950 return ret;
951}
952
953void starfive_aes_unregister_algs(void)
954{
955 crypto_engine_unregister_aeads(algs: aead_algs, ARRAY_SIZE(aead_algs));
956 crypto_engine_unregister_skciphers(algs: skcipher_algs, ARRAY_SIZE(skcipher_algs));
957}
958

source code of linux/drivers/crypto/starfive/jh7110-aes.c