1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Xilinx ZynqMP AES Driver. |
4 | * Copyright (c) 2020 Xilinx Inc. |
5 | */ |
6 | |
7 | #include <crypto/aes.h> |
8 | #include <crypto/engine.h> |
9 | #include <crypto/gcm.h> |
10 | #include <crypto/internal/aead.h> |
11 | #include <crypto/scatterwalk.h> |
12 | #include <linux/dma-mapping.h> |
13 | #include <linux/err.h> |
14 | #include <linux/firmware/xlnx-zynqmp.h> |
15 | #include <linux/kernel.h> |
16 | #include <linux/module.h> |
17 | #include <linux/mod_devicetable.h> |
18 | #include <linux/platform_device.h> |
19 | #include <linux/string.h> |
20 | |
21 | #define ZYNQMP_DMA_BIT_MASK 32U |
22 | |
23 | #define ZYNQMP_AES_KEY_SIZE AES_KEYSIZE_256 |
24 | #define ZYNQMP_AES_AUTH_SIZE 16U |
25 | #define ZYNQMP_KEY_SRC_SEL_KEY_LEN 1U |
26 | #define ZYNQMP_AES_BLK_SIZE 1U |
27 | #define ZYNQMP_AES_MIN_INPUT_BLK_SIZE 4U |
28 | #define ZYNQMP_AES_WORD_LEN 4U |
29 | |
30 | #define ZYNQMP_AES_GCM_TAG_MISMATCH_ERR 0x01 |
31 | #define ZYNQMP_AES_WRONG_KEY_SRC_ERR 0x13 |
32 | #define ZYNQMP_AES_PUF_NOT_PROGRAMMED 0xE300 |
33 | |
34 | enum zynqmp_aead_op { |
35 | ZYNQMP_AES_DECRYPT = 0, |
36 | ZYNQMP_AES_ENCRYPT |
37 | }; |
38 | |
39 | enum zynqmp_aead_keysrc { |
40 | ZYNQMP_AES_KUP_KEY = 0, |
41 | ZYNQMP_AES_DEV_KEY, |
42 | ZYNQMP_AES_PUF_KEY |
43 | }; |
44 | |
45 | struct zynqmp_aead_drv_ctx { |
46 | union { |
47 | struct aead_engine_alg aead; |
48 | } alg; |
49 | struct device *dev; |
50 | struct crypto_engine *engine; |
51 | }; |
52 | |
53 | struct zynqmp_aead_hw_req { |
54 | u64 src; |
55 | u64 iv; |
56 | u64 key; |
57 | u64 dst; |
58 | u64 size; |
59 | u64 op; |
60 | u64 keysrc; |
61 | }; |
62 | |
63 | struct zynqmp_aead_tfm_ctx { |
64 | struct device *dev; |
65 | u8 key[ZYNQMP_AES_KEY_SIZE]; |
66 | u8 *iv; |
67 | u32 keylen; |
68 | u32 authsize; |
69 | enum zynqmp_aead_keysrc keysrc; |
70 | struct crypto_aead *fbk_cipher; |
71 | }; |
72 | |
73 | struct zynqmp_aead_req_ctx { |
74 | enum zynqmp_aead_op op; |
75 | }; |
76 | |
77 | static int zynqmp_aes_aead_cipher(struct aead_request *req) |
78 | { |
79 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
80 | struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(tfm: aead); |
81 | struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); |
82 | struct device *dev = tfm_ctx->dev; |
83 | struct zynqmp_aead_hw_req *hwreq; |
84 | dma_addr_t dma_addr_data, dma_addr_hw_req; |
85 | unsigned int data_size; |
86 | unsigned int status; |
87 | int ret; |
88 | size_t dma_size; |
89 | char *kbuf; |
90 | int err; |
91 | |
92 | if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY) |
93 | dma_size = req->cryptlen + ZYNQMP_AES_KEY_SIZE |
94 | + GCM_AES_IV_SIZE; |
95 | else |
96 | dma_size = req->cryptlen + GCM_AES_IV_SIZE; |
97 | |
98 | kbuf = dma_alloc_coherent(dev, size: dma_size, dma_handle: &dma_addr_data, GFP_KERNEL); |
99 | if (!kbuf) |
100 | return -ENOMEM; |
101 | |
102 | hwreq = dma_alloc_coherent(dev, size: sizeof(struct zynqmp_aead_hw_req), |
103 | dma_handle: &dma_addr_hw_req, GFP_KERNEL); |
104 | if (!hwreq) { |
105 | dma_free_coherent(dev, size: dma_size, cpu_addr: kbuf, dma_handle: dma_addr_data); |
106 | return -ENOMEM; |
107 | } |
108 | |
109 | data_size = req->cryptlen; |
110 | scatterwalk_map_and_copy(buf: kbuf, sg: req->src, start: 0, nbytes: req->cryptlen, out: 0); |
111 | memcpy(kbuf + data_size, req->iv, GCM_AES_IV_SIZE); |
112 | |
113 | hwreq->src = dma_addr_data; |
114 | hwreq->dst = dma_addr_data; |
115 | hwreq->iv = hwreq->src + data_size; |
116 | hwreq->keysrc = tfm_ctx->keysrc; |
117 | hwreq->op = rq_ctx->op; |
118 | |
119 | if (hwreq->op == ZYNQMP_AES_ENCRYPT) |
120 | hwreq->size = data_size; |
121 | else |
122 | hwreq->size = data_size - ZYNQMP_AES_AUTH_SIZE; |
123 | |
124 | if (hwreq->keysrc == ZYNQMP_AES_KUP_KEY) { |
125 | memcpy(kbuf + data_size + GCM_AES_IV_SIZE, |
126 | tfm_ctx->key, ZYNQMP_AES_KEY_SIZE); |
127 | |
128 | hwreq->key = hwreq->src + data_size + GCM_AES_IV_SIZE; |
129 | } else { |
130 | hwreq->key = 0; |
131 | } |
132 | |
133 | ret = zynqmp_pm_aes_engine(address: dma_addr_hw_req, out: &status); |
134 | |
135 | if (ret) { |
136 | dev_err(dev, "ERROR: AES PM API failed\n" ); |
137 | err = ret; |
138 | } else if (status) { |
139 | switch (status) { |
140 | case ZYNQMP_AES_GCM_TAG_MISMATCH_ERR: |
141 | dev_err(dev, "ERROR: Gcm Tag mismatch\n" ); |
142 | break; |
143 | case ZYNQMP_AES_WRONG_KEY_SRC_ERR: |
144 | dev_err(dev, "ERROR: Wrong KeySrc, enable secure mode\n" ); |
145 | break; |
146 | case ZYNQMP_AES_PUF_NOT_PROGRAMMED: |
147 | dev_err(dev, "ERROR: PUF is not registered\n" ); |
148 | break; |
149 | default: |
150 | dev_err(dev, "ERROR: Unknown error\n" ); |
151 | break; |
152 | } |
153 | err = -status; |
154 | } else { |
155 | if (hwreq->op == ZYNQMP_AES_ENCRYPT) |
156 | data_size = data_size + ZYNQMP_AES_AUTH_SIZE; |
157 | else |
158 | data_size = data_size - ZYNQMP_AES_AUTH_SIZE; |
159 | |
160 | sg_copy_from_buffer(sgl: req->dst, nents: sg_nents(sg: req->dst), |
161 | buf: kbuf, buflen: data_size); |
162 | err = 0; |
163 | } |
164 | |
165 | if (kbuf) { |
166 | memzero_explicit(s: kbuf, count: dma_size); |
167 | dma_free_coherent(dev, size: dma_size, cpu_addr: kbuf, dma_handle: dma_addr_data); |
168 | } |
169 | if (hwreq) { |
170 | memzero_explicit(s: hwreq, count: sizeof(struct zynqmp_aead_hw_req)); |
171 | dma_free_coherent(dev, size: sizeof(struct zynqmp_aead_hw_req), |
172 | cpu_addr: hwreq, dma_handle: dma_addr_hw_req); |
173 | } |
174 | return err; |
175 | } |
176 | |
177 | static int zynqmp_fallback_check(struct zynqmp_aead_tfm_ctx *tfm_ctx, |
178 | struct aead_request *req) |
179 | { |
180 | int need_fallback = 0; |
181 | struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); |
182 | |
183 | if (tfm_ctx->authsize != ZYNQMP_AES_AUTH_SIZE) |
184 | need_fallback = 1; |
185 | |
186 | if (tfm_ctx->keysrc == ZYNQMP_AES_KUP_KEY && |
187 | tfm_ctx->keylen != ZYNQMP_AES_KEY_SIZE) { |
188 | need_fallback = 1; |
189 | } |
190 | if (req->assoclen != 0 || |
191 | req->cryptlen < ZYNQMP_AES_MIN_INPUT_BLK_SIZE) { |
192 | need_fallback = 1; |
193 | } |
194 | if ((req->cryptlen % ZYNQMP_AES_WORD_LEN) != 0) |
195 | need_fallback = 1; |
196 | |
197 | if (rq_ctx->op == ZYNQMP_AES_DECRYPT && |
198 | req->cryptlen <= ZYNQMP_AES_AUTH_SIZE) { |
199 | need_fallback = 1; |
200 | } |
201 | return need_fallback; |
202 | } |
203 | |
204 | static int zynqmp_handle_aes_req(struct crypto_engine *engine, |
205 | void *req) |
206 | { |
207 | struct aead_request *areq = |
208 | container_of(req, struct aead_request, base); |
209 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
210 | struct zynqmp_aead_tfm_ctx *tfm_ctx = crypto_aead_ctx(tfm: aead); |
211 | struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req: areq); |
212 | struct aead_request *subreq = aead_request_ctx(req); |
213 | int need_fallback; |
214 | int err; |
215 | |
216 | need_fallback = zynqmp_fallback_check(tfm_ctx, req: areq); |
217 | |
218 | if (need_fallback) { |
219 | aead_request_set_tfm(req: subreq, tfm: tfm_ctx->fbk_cipher); |
220 | |
221 | aead_request_set_callback(req: subreq, flags: areq->base.flags, |
222 | NULL, NULL); |
223 | aead_request_set_crypt(req: subreq, src: areq->src, dst: areq->dst, |
224 | cryptlen: areq->cryptlen, iv: areq->iv); |
225 | aead_request_set_ad(req: subreq, assoclen: areq->assoclen); |
226 | if (rq_ctx->op == ZYNQMP_AES_ENCRYPT) |
227 | err = crypto_aead_encrypt(req: subreq); |
228 | else |
229 | err = crypto_aead_decrypt(req: subreq); |
230 | } else { |
231 | err = zynqmp_aes_aead_cipher(req: areq); |
232 | } |
233 | |
234 | local_bh_disable(); |
235 | crypto_finalize_aead_request(engine, req: areq, err); |
236 | local_bh_enable(); |
237 | |
238 | return 0; |
239 | } |
240 | |
241 | static int zynqmp_aes_aead_setkey(struct crypto_aead *aead, const u8 *key, |
242 | unsigned int keylen) |
243 | { |
244 | struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead); |
245 | struct zynqmp_aead_tfm_ctx *tfm_ctx = |
246 | (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); |
247 | unsigned char keysrc; |
248 | |
249 | if (keylen == ZYNQMP_KEY_SRC_SEL_KEY_LEN) { |
250 | keysrc = *key; |
251 | if (keysrc == ZYNQMP_AES_KUP_KEY || |
252 | keysrc == ZYNQMP_AES_DEV_KEY || |
253 | keysrc == ZYNQMP_AES_PUF_KEY) { |
254 | tfm_ctx->keysrc = (enum zynqmp_aead_keysrc)keysrc; |
255 | } else { |
256 | tfm_ctx->keylen = keylen; |
257 | } |
258 | } else { |
259 | tfm_ctx->keylen = keylen; |
260 | if (keylen == ZYNQMP_AES_KEY_SIZE) { |
261 | tfm_ctx->keysrc = ZYNQMP_AES_KUP_KEY; |
262 | memcpy(tfm_ctx->key, key, keylen); |
263 | } |
264 | } |
265 | |
266 | tfm_ctx->fbk_cipher->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK; |
267 | tfm_ctx->fbk_cipher->base.crt_flags |= (aead->base.crt_flags & |
268 | CRYPTO_TFM_REQ_MASK); |
269 | |
270 | return crypto_aead_setkey(tfm: tfm_ctx->fbk_cipher, key, keylen); |
271 | } |
272 | |
273 | static int zynqmp_aes_aead_setauthsize(struct crypto_aead *aead, |
274 | unsigned int authsize) |
275 | { |
276 | struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead); |
277 | struct zynqmp_aead_tfm_ctx *tfm_ctx = |
278 | (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); |
279 | |
280 | tfm_ctx->authsize = authsize; |
281 | return crypto_aead_setauthsize(tfm: tfm_ctx->fbk_cipher, authsize); |
282 | } |
283 | |
284 | static int zynqmp_aes_aead_encrypt(struct aead_request *req) |
285 | { |
286 | struct zynqmp_aead_drv_ctx *drv_ctx; |
287 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
288 | struct aead_alg *alg = crypto_aead_alg(tfm: aead); |
289 | struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); |
290 | |
291 | rq_ctx->op = ZYNQMP_AES_ENCRYPT; |
292 | drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); |
293 | |
294 | return crypto_transfer_aead_request_to_engine(engine: drv_ctx->engine, req); |
295 | } |
296 | |
297 | static int zynqmp_aes_aead_decrypt(struct aead_request *req) |
298 | { |
299 | struct zynqmp_aead_drv_ctx *drv_ctx; |
300 | struct crypto_aead *aead = crypto_aead_reqtfm(req); |
301 | struct aead_alg *alg = crypto_aead_alg(tfm: aead); |
302 | struct zynqmp_aead_req_ctx *rq_ctx = aead_request_ctx(req); |
303 | |
304 | rq_ctx->op = ZYNQMP_AES_DECRYPT; |
305 | drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); |
306 | |
307 | return crypto_transfer_aead_request_to_engine(engine: drv_ctx->engine, req); |
308 | } |
309 | |
310 | static int zynqmp_aes_aead_init(struct crypto_aead *aead) |
311 | { |
312 | struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead); |
313 | struct zynqmp_aead_tfm_ctx *tfm_ctx = |
314 | (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); |
315 | struct zynqmp_aead_drv_ctx *drv_ctx; |
316 | struct aead_alg *alg = crypto_aead_alg(tfm: aead); |
317 | |
318 | drv_ctx = container_of(alg, struct zynqmp_aead_drv_ctx, alg.aead.base); |
319 | tfm_ctx->dev = drv_ctx->dev; |
320 | |
321 | tfm_ctx->fbk_cipher = crypto_alloc_aead(alg_name: drv_ctx->alg.aead.base.base.cra_name, |
322 | type: 0, |
323 | CRYPTO_ALG_NEED_FALLBACK); |
324 | |
325 | if (IS_ERR(ptr: tfm_ctx->fbk_cipher)) { |
326 | pr_err("%s() Error: failed to allocate fallback for %s\n" , |
327 | __func__, drv_ctx->alg.aead.base.base.cra_name); |
328 | return PTR_ERR(ptr: tfm_ctx->fbk_cipher); |
329 | } |
330 | |
331 | crypto_aead_set_reqsize(aead, |
332 | max(sizeof(struct zynqmp_aead_req_ctx), |
333 | sizeof(struct aead_request) + |
334 | crypto_aead_reqsize(tfm_ctx->fbk_cipher))); |
335 | return 0; |
336 | } |
337 | |
338 | static void zynqmp_aes_aead_exit(struct crypto_aead *aead) |
339 | { |
340 | struct crypto_tfm *tfm = crypto_aead_tfm(tfm: aead); |
341 | struct zynqmp_aead_tfm_ctx *tfm_ctx = |
342 | (struct zynqmp_aead_tfm_ctx *)crypto_tfm_ctx(tfm); |
343 | |
344 | if (tfm_ctx->fbk_cipher) { |
345 | crypto_free_aead(tfm: tfm_ctx->fbk_cipher); |
346 | tfm_ctx->fbk_cipher = NULL; |
347 | } |
348 | memzero_explicit(s: tfm_ctx, count: sizeof(struct zynqmp_aead_tfm_ctx)); |
349 | } |
350 | |
351 | static struct zynqmp_aead_drv_ctx aes_drv_ctx = { |
352 | .alg.aead.base = { |
353 | .setkey = zynqmp_aes_aead_setkey, |
354 | .setauthsize = zynqmp_aes_aead_setauthsize, |
355 | .encrypt = zynqmp_aes_aead_encrypt, |
356 | .decrypt = zynqmp_aes_aead_decrypt, |
357 | .init = zynqmp_aes_aead_init, |
358 | .exit = zynqmp_aes_aead_exit, |
359 | .ivsize = GCM_AES_IV_SIZE, |
360 | .maxauthsize = ZYNQMP_AES_AUTH_SIZE, |
361 | .base = { |
362 | .cra_name = "gcm(aes)" , |
363 | .cra_driver_name = "xilinx-zynqmp-aes-gcm" , |
364 | .cra_priority = 200, |
365 | .cra_flags = CRYPTO_ALG_TYPE_AEAD | |
366 | CRYPTO_ALG_ASYNC | |
367 | CRYPTO_ALG_ALLOCATES_MEMORY | |
368 | CRYPTO_ALG_KERN_DRIVER_ONLY | |
369 | CRYPTO_ALG_NEED_FALLBACK, |
370 | .cra_blocksize = ZYNQMP_AES_BLK_SIZE, |
371 | .cra_ctxsize = sizeof(struct zynqmp_aead_tfm_ctx), |
372 | .cra_module = THIS_MODULE, |
373 | } |
374 | }, |
375 | .alg.aead.op = { |
376 | .do_one_request = zynqmp_handle_aes_req, |
377 | }, |
378 | }; |
379 | |
380 | static int zynqmp_aes_aead_probe(struct platform_device *pdev) |
381 | { |
382 | struct device *dev = &pdev->dev; |
383 | int err; |
384 | |
385 | /* ZynqMP AES driver supports only one instance */ |
386 | if (!aes_drv_ctx.dev) |
387 | aes_drv_ctx.dev = dev; |
388 | else |
389 | return -ENODEV; |
390 | |
391 | err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(ZYNQMP_DMA_BIT_MASK)); |
392 | if (err < 0) { |
393 | dev_err(dev, "No usable DMA configuration\n" ); |
394 | return err; |
395 | } |
396 | |
397 | aes_drv_ctx.engine = crypto_engine_alloc_init(dev, rt: 1); |
398 | if (!aes_drv_ctx.engine) { |
399 | dev_err(dev, "Cannot alloc AES engine\n" ); |
400 | err = -ENOMEM; |
401 | goto err_engine; |
402 | } |
403 | |
404 | err = crypto_engine_start(engine: aes_drv_ctx.engine); |
405 | if (err) { |
406 | dev_err(dev, "Cannot start AES engine\n" ); |
407 | goto err_engine; |
408 | } |
409 | |
410 | err = crypto_engine_register_aead(alg: &aes_drv_ctx.alg.aead); |
411 | if (err < 0) { |
412 | dev_err(dev, "Failed to register AEAD alg.\n" ); |
413 | goto err_aead; |
414 | } |
415 | return 0; |
416 | |
417 | err_aead: |
418 | crypto_engine_unregister_aead(alg: &aes_drv_ctx.alg.aead); |
419 | |
420 | err_engine: |
421 | if (aes_drv_ctx.engine) |
422 | crypto_engine_exit(engine: aes_drv_ctx.engine); |
423 | |
424 | return err; |
425 | } |
426 | |
427 | static void zynqmp_aes_aead_remove(struct platform_device *pdev) |
428 | { |
429 | crypto_engine_exit(engine: aes_drv_ctx.engine); |
430 | crypto_engine_unregister_aead(alg: &aes_drv_ctx.alg.aead); |
431 | } |
432 | |
433 | static const struct of_device_id zynqmp_aes_dt_ids[] = { |
434 | { .compatible = "xlnx,zynqmp-aes" }, |
435 | { /* sentinel */ } |
436 | }; |
437 | MODULE_DEVICE_TABLE(of, zynqmp_aes_dt_ids); |
438 | |
439 | static struct platform_driver zynqmp_aes_driver = { |
440 | .probe = zynqmp_aes_aead_probe, |
441 | .remove_new = zynqmp_aes_aead_remove, |
442 | .driver = { |
443 | .name = "zynqmp-aes" , |
444 | .of_match_table = zynqmp_aes_dt_ids, |
445 | }, |
446 | }; |
447 | |
448 | module_platform_driver(zynqmp_aes_driver); |
449 | MODULE_LICENSE("GPL" ); |
450 | |