1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* AMD Cryptographic Coprocessor (CCP) driver
4	*
5	* Copyright (C) 2013-2019 Advanced Micro Devices, Inc.
6	*
7	* Author: Tom Lendacky <thomas.lendacky@amd.com>
8	* Author: Gary R Hook <gary.hook@amd.com>
9	*/
10
11	#include <linux/dma-mapping.h>
12	#include <linux/module.h>
13	#include <linux/kernel.h>
14	#include <linux/interrupt.h>
15	#include <crypto/scatterwalk.h>
16	#include <crypto/des.h>
17	#include <linux/ccp.h>
18
19	#include "ccp-dev.h"
20
21	/* SHA initial context values */
22	static const __be32 ccp_sha1_init[SHA1_DIGEST_SIZE / sizeof(__be32)] = {
23	cpu_to_be32(SHA1_H0), cpu_to_be32(SHA1_H1),
24	cpu_to_be32(SHA1_H2), cpu_to_be32(SHA1_H3),
25	cpu_to_be32(SHA1_H4),
26	};
27
28	static const __be32 ccp_sha224_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
29	cpu_to_be32(SHA224_H0), cpu_to_be32(SHA224_H1),
30	cpu_to_be32(SHA224_H2), cpu_to_be32(SHA224_H3),
31	cpu_to_be32(SHA224_H4), cpu_to_be32(SHA224_H5),
32	cpu_to_be32(SHA224_H6), cpu_to_be32(SHA224_H7),
33	};
34
35	static const __be32 ccp_sha256_init[SHA256_DIGEST_SIZE / sizeof(__be32)] = {
36	cpu_to_be32(SHA256_H0), cpu_to_be32(SHA256_H1),
37	cpu_to_be32(SHA256_H2), cpu_to_be32(SHA256_H3),
38	cpu_to_be32(SHA256_H4), cpu_to_be32(SHA256_H5),
39	cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7),
40	};
41
42	static const __be64 ccp_sha384_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
43	cpu_to_be64(SHA384_H0), cpu_to_be64(SHA384_H1),
44	cpu_to_be64(SHA384_H2), cpu_to_be64(SHA384_H3),
45	cpu_to_be64(SHA384_H4), cpu_to_be64(SHA384_H5),
46	cpu_to_be64(SHA384_H6), cpu_to_be64(SHA384_H7),
47	};
48
49	static const __be64 ccp_sha512_init[SHA512_DIGEST_SIZE / sizeof(__be64)] = {
50	cpu_to_be64(SHA512_H0), cpu_to_be64(SHA512_H1),
51	cpu_to_be64(SHA512_H2), cpu_to_be64(SHA512_H3),
52	cpu_to_be64(SHA512_H4), cpu_to_be64(SHA512_H5),
53	cpu_to_be64(SHA512_H6), cpu_to_be64(SHA512_H7),
54	};
55
56	#define CCP_NEW_JOBID(ccp) ((ccp->vdata->version == CCP_VERSION(3, 0)) ? \
57	ccp_gen_jobid(ccp) : 0)
58
59	static u32 ccp_gen_jobid(struct ccp_device *ccp)
60	{
61	return atomic_inc_return(v: &ccp->current_id) & CCP_JOBID_MASK;
62	}
63
64	static void ccp_sg_free(struct ccp_sg_workarea *wa)
65	{
66	if (wa->dma_count)
67	dma_unmap_sg(wa->dma_dev, wa->dma_sg_head, wa->nents, wa->dma_dir);
68
69	wa->dma_count = 0;
70	}
71
72	static int ccp_init_sg_workarea(struct ccp_sg_workarea *wa, struct device *dev,
73	struct scatterlist *sg, u64 len,
74	enum dma_data_direction dma_dir)
75	{
76	memset(wa, 0, sizeof(*wa));
77
78	wa->sg = sg;
79	if (!sg)
80	return 0;
81
82	wa->nents = sg_nents_for_len(sg, len);
83	if (wa->nents < 0)
84	return wa->nents;
85
86	wa->bytes_left = len;
87	wa->sg_used = 0;
88
89	if (len == 0)
90	return 0;
91
92	if (dma_dir == DMA_NONE)
93	return 0;
94
95	wa->dma_sg = sg;
96	wa->dma_sg_head = sg;
97	wa->dma_dev = dev;
98	wa->dma_dir = dma_dir;
99	wa->dma_count = dma_map_sg(dev, sg, wa->nents, dma_dir);
100	if (!wa->dma_count)
101	return -ENOMEM;
102
103	return 0;
104	}
105
106	static void ccp_update_sg_workarea(struct ccp_sg_workarea *wa, unsigned int len)
107	{
108	unsigned int nbytes = min_t(u64, len, wa->bytes_left);
109	unsigned int sg_combined_len = 0;
110
111	if (!wa->sg)
112	return;
113
114	wa->sg_used += nbytes;
115	wa->bytes_left -= nbytes;
116	if (wa->sg_used == sg_dma_len(wa->dma_sg)) {
117	/* Advance to the next DMA scatterlist entry */
118	wa->dma_sg = sg_next(wa->dma_sg);
119
120	/* In the case that the DMA mapped scatterlist has entries
121	* that have been merged, the non-DMA mapped scatterlist
122	* must be advanced multiple times for each merged entry.
123	* This ensures that the current non-DMA mapped entry
124	* corresponds to the current DMA mapped entry.
125	*/
126	do {
127	sg_combined_len += wa->sg->length;
128	wa->sg = sg_next(wa->sg);
129	} while (wa->sg_used > sg_combined_len);
130
131	wa->sg_used = 0;
132	}
133	}
134
135	static void ccp_dm_free(struct ccp_dm_workarea *wa)
136	{
137	if (wa->length <= CCP_DMAPOOL_MAX_SIZE) {
138	if (wa->address)
139	dma_pool_free(pool: wa->dma_pool, vaddr: wa->address,
140	addr: wa->dma.address);
141	} else {
142	if (wa->dma.address)
143	dma_unmap_single(wa->dev, wa->dma.address, wa->length,
144	wa->dma.dir);
145	kfree(objp: wa->address);
146	}
147
148	wa->address = NULL;
149	wa->dma.address = 0;
150	}
151
152	static int ccp_init_dm_workarea(struct ccp_dm_workarea *wa,
153	struct ccp_cmd_queue *cmd_q,
154	unsigned int len,
155	enum dma_data_direction dir)
156	{
157	memset(wa, 0, sizeof(*wa));
158
159	if (!len)
160	return 0;
161
162	wa->dev = cmd_q->ccp->dev;
163	wa->length = len;
164
165	if (len <= CCP_DMAPOOL_MAX_SIZE) {
166	wa->dma_pool = cmd_q->dma_pool;
167
168	wa->address = dma_pool_zalloc(pool: wa->dma_pool, GFP_KERNEL,
169	handle: &wa->dma.address);
170	if (!wa->address)
171	return -ENOMEM;
172
173	wa->dma.length = CCP_DMAPOOL_MAX_SIZE;
174
175	} else {
176	wa->address = kzalloc(size: len, GFP_KERNEL);
177	if (!wa->address)
178	return -ENOMEM;
179
180	wa->dma.address = dma_map_single(wa->dev, wa->address, len,
181	dir);
182	if (dma_mapping_error(dev: wa->dev, dma_addr: wa->dma.address)) {
183	kfree(objp: wa->address);
184	wa->address = NULL;
185	return -ENOMEM;
186	}
187
188	wa->dma.length = len;
189	}
190	wa->dma.dir = dir;
191
192	return 0;
193	}
194
195	static int ccp_set_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
196	struct scatterlist *sg, unsigned int sg_offset,
197	unsigned int len)
198	{
199	WARN_ON(!wa->address);
200
201	if (len > (wa->length - wa_offset))
202	return -EINVAL;
203
204	scatterwalk_map_and_copy(buf: wa->address + wa_offset, sg, start: sg_offset, nbytes: len,
205	out: 0);
206	return 0;
207	}
208
209	static void ccp_get_dm_area(struct ccp_dm_workarea *wa, unsigned int wa_offset,
210	struct scatterlist *sg, unsigned int sg_offset,
211	unsigned int len)
212	{
213	WARN_ON(!wa->address);
214
215	scatterwalk_map_and_copy(buf: wa->address + wa_offset, sg, start: sg_offset, nbytes: len,
216	out: 1);
217	}
218
219	static int ccp_reverse_set_dm_area(struct ccp_dm_workarea *wa,
220	unsigned int wa_offset,
221	struct scatterlist *sg,
222	unsigned int sg_offset,
223	unsigned int len)
224	{
225	u8 *p, *q;
226	int rc;
227
228	rc = ccp_set_dm_area(wa, wa_offset, sg, sg_offset, len);
229	if (rc)
230	return rc;
231
232	p = wa->address + wa_offset;
233	q = p + len - 1;
234	while (p < q) {
235	*p = *p ^ *q;
236	*q = *p ^ *q;
237	*p = *p ^ *q;
238	p++;
239	q--;
240	}
241	return 0;
242	}
243
244	static void ccp_reverse_get_dm_area(struct ccp_dm_workarea *wa,
245	unsigned int wa_offset,
246	struct scatterlist *sg,
247	unsigned int sg_offset,
248	unsigned int len)
249	{
250	u8 *p, *q;
251
252	p = wa->address + wa_offset;
253	q = p + len - 1;
254	while (p < q) {
255	*p = *p ^ *q;
256	*q = *p ^ *q;
257	*p = *p ^ *q;
258	p++;
259	q--;
260	}
261
262	ccp_get_dm_area(wa, wa_offset, sg, sg_offset, len);
263	}
264
265	static void ccp_free_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q)
266	{
267	ccp_dm_free(wa: &data->dm_wa);
268	ccp_sg_free(wa: &data->sg_wa);
269	}
270
271	static int ccp_init_data(struct ccp_data *data, struct ccp_cmd_queue *cmd_q,
272	struct scatterlist *sg, u64 sg_len,
273	unsigned int dm_len,
274	enum dma_data_direction dir)
275	{
276	int ret;
277
278	memset(data, 0, sizeof(*data));
279
280	ret = ccp_init_sg_workarea(wa: &data->sg_wa, dev: cmd_q->ccp->dev, sg, len: sg_len,
281	dma_dir: dir);
282	if (ret)
283	goto e_err;
284
285	ret = ccp_init_dm_workarea(wa: &data->dm_wa, cmd_q, len: dm_len, dir);
286	if (ret)
287	goto e_err;
288
289	return 0;
290
291	e_err:
292	ccp_free_data(data, cmd_q);
293
294	return ret;
295	}
296
297	static unsigned int ccp_queue_buf(struct ccp_data *data, unsigned int from)
298	{
299	struct ccp_sg_workarea *sg_wa = &data->sg_wa;
300	struct ccp_dm_workarea *dm_wa = &data->dm_wa;
301	unsigned int buf_count, nbytes;
302
303	/* Clear the buffer if setting it */
304	if (!from)
305	memset(dm_wa->address, 0, dm_wa->length);
306
307	if (!sg_wa->sg)
308	return 0;
309
310	/* Perform the copy operation
311	* nbytes will always be <= UINT_MAX because dm_wa->length is
312	* an unsigned int
313	*/
314	nbytes = min_t(u64, sg_wa->bytes_left, dm_wa->length);
315	scatterwalk_map_and_copy(buf: dm_wa->address, sg: sg_wa->sg, start: sg_wa->sg_used,
316	nbytes, out: from);
317
318	/* Update the structures and generate the count */
319	buf_count = 0;
320	while (sg_wa->bytes_left && (buf_count < dm_wa->length)) {
321	nbytes = min(sg_dma_len(sg_wa->dma_sg) - sg_wa->sg_used,
322	dm_wa->length - buf_count);
323	nbytes = min_t(u64, sg_wa->bytes_left, nbytes);
324
325	buf_count += nbytes;
326	ccp_update_sg_workarea(wa: sg_wa, len: nbytes);
327	}
328
329	return buf_count;
330	}
331
332	static unsigned int ccp_fill_queue_buf(struct ccp_data *data)
333	{
334	return ccp_queue_buf(data, from: 0);
335	}
336
337	static unsigned int ccp_empty_queue_buf(struct ccp_data *data)
338	{
339	return ccp_queue_buf(data, from: 1);
340	}
341
342	static void ccp_prepare_data(struct ccp_data *src, struct ccp_data *dst,
343	struct ccp_op *op, unsigned int block_size,
344	bool blocksize_op)
345	{
346	unsigned int sg_src_len, sg_dst_len, op_len;
347
348	/* The CCP can only DMA from/to one address each per operation. This
349	* requires that we find the smallest DMA area between the source
350	* and destination. The resulting len values will always be <= UINT_MAX
351	* because the dma length is an unsigned int.
352	*/
353	sg_src_len = sg_dma_len(src->sg_wa.dma_sg) - src->sg_wa.sg_used;
354	sg_src_len = min_t(u64, src->sg_wa.bytes_left, sg_src_len);
355
356	if (dst) {
357	sg_dst_len = sg_dma_len(dst->sg_wa.dma_sg) - dst->sg_wa.sg_used;
358	sg_dst_len = min_t(u64, src->sg_wa.bytes_left, sg_dst_len);
359	op_len = min(sg_src_len, sg_dst_len);
360	} else {
361	op_len = sg_src_len;
362	}
363
364	/* The data operation length will be at least block_size in length
365	* or the smaller of available sg room remaining for the source or
366	* the destination
367	*/
368	op_len = max(op_len, block_size);
369
370	/* Unless we have to buffer data, there's no reason to wait */
371	op->soc = 0;
372
373	if (sg_src_len < block_size) {
374	/* Not enough data in the sg element, so it
375	* needs to be buffered into a blocksize chunk
376	*/
377	int cp_len = ccp_fill_queue_buf(data: src);
378
379	op->soc = 1;
380	op->src.u.dma.address = src->dm_wa.dma.address;
381	op->src.u.dma.offset = 0;
382	op->src.u.dma.length = (blocksize_op) ? block_size : cp_len;
383	} else {
384	/* Enough data in the sg element, but we need to
385	* adjust for any previously copied data
386	*/
387	op->src.u.dma.address = sg_dma_address(src->sg_wa.dma_sg);
388	op->src.u.dma.offset = src->sg_wa.sg_used;
389	op->src.u.dma.length = op_len & ~(block_size - 1);
390
391	ccp_update_sg_workarea(wa: &src->sg_wa, len: op->src.u.dma.length);
392	}
393
394	if (dst) {
395	if (sg_dst_len < block_size) {
396	/* Not enough room in the sg element or we're on the
397	* last piece of data (when using padding), so the
398	* output needs to be buffered into a blocksize chunk
399	*/
400	op->soc = 1;
401	op->dst.u.dma.address = dst->dm_wa.dma.address;
402	op->dst.u.dma.offset = 0;
403	op->dst.u.dma.length = op->src.u.dma.length;
404	} else {
405	/* Enough room in the sg element, but we need to
406	* adjust for any previously used area
407	*/
408	op->dst.u.dma.address = sg_dma_address(dst->sg_wa.dma_sg);
409	op->dst.u.dma.offset = dst->sg_wa.sg_used;
410	op->dst.u.dma.length = op->src.u.dma.length;
411	}
412	}
413	}
414
415	static void ccp_process_data(struct ccp_data *src, struct ccp_data *dst,
416	struct ccp_op *op)
417	{
418	op->init = 0;
419
420	if (dst) {
421	if (op->dst.u.dma.address == dst->dm_wa.dma.address)
422	ccp_empty_queue_buf(data: dst);
423	else
424	ccp_update_sg_workarea(wa: &dst->sg_wa,
425	len: op->dst.u.dma.length);
426	}
427	}
428
429	static int ccp_copy_to_from_sb(struct ccp_cmd_queue *cmd_q,
430	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
431	u32 byte_swap, bool from)
432	{
433	struct ccp_op op;
434
435	memset(&op, 0, sizeof(op));
436
437	op.cmd_q = cmd_q;
438	op.jobid = jobid;
439	op.eom = 1;
440
441	if (from) {
442	op.soc = 1;
443	op.src.type = CCP_MEMTYPE_SB;
444	op.src.u.sb = sb;
445	op.dst.type = CCP_MEMTYPE_SYSTEM;
446	op.dst.u.dma.address = wa->dma.address;
447	op.dst.u.dma.length = wa->length;
448	} else {
449	op.src.type = CCP_MEMTYPE_SYSTEM;
450	op.src.u.dma.address = wa->dma.address;
451	op.src.u.dma.length = wa->length;
452	op.dst.type = CCP_MEMTYPE_SB;
453	op.dst.u.sb = sb;
454	}
455
456	op.u.passthru.byte_swap = byte_swap;
457
458	return cmd_q->ccp->vdata->perform->passthru(&op);
459	}
460
461	static int ccp_copy_to_sb(struct ccp_cmd_queue *cmd_q,
462	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
463	u32 byte_swap)
464	{
465	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, from: false);
466	}
467
468	static int ccp_copy_from_sb(struct ccp_cmd_queue *cmd_q,
469	struct ccp_dm_workarea *wa, u32 jobid, u32 sb,
470	u32 byte_swap)
471	{
472	return ccp_copy_to_from_sb(cmd_q, wa, jobid, sb, byte_swap, from: true);
473	}
474
475	static noinline_for_stack int
476	ccp_run_aes_cmac_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
477	{
478	struct ccp_aes_engine *aes = &cmd->u.aes;
479	struct ccp_dm_workarea key, ctx;
480	struct ccp_data src;
481	struct ccp_op op;
482	unsigned int dm_offset;
483	int ret;
484
485	if (!((aes->key_len == AES_KEYSIZE_128) ||
486	(aes->key_len == AES_KEYSIZE_192) ||
487	(aes->key_len == AES_KEYSIZE_256)))
488	return -EINVAL;
489
490	if (aes->src_len & (AES_BLOCK_SIZE - 1))
491	return -EINVAL;
492
493	if (aes->iv_len != AES_BLOCK_SIZE)
494	return -EINVAL;
495
496	if (!aes->key || !aes->iv || !aes->src)
497	return -EINVAL;
498
499	if (aes->cmac_final) {
500	if (aes->cmac_key_len != AES_BLOCK_SIZE)
501	return -EINVAL;
502
503	if (!aes->cmac_key)
504	return -EINVAL;
505	}
506
507	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
508	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
509
510	ret = -EIO;
511	memset(&op, 0, sizeof(op));
512	op.cmd_q = cmd_q;
513	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
514	op.sb_key = cmd_q->sb_key;
515	op.sb_ctx = cmd_q->sb_ctx;
516	op.init = 1;
517	op.u.aes.type = aes->type;
518	op.u.aes.mode = aes->mode;
519	op.u.aes.action = aes->action;
520
521	/* All supported key sizes fit in a single (32-byte) SB entry
522	* and must be in little endian format. Use the 256-bit byte
523	* swap passthru option to convert from big endian to little
524	* endian.
525	*/
526	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
527	CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
528	dir: DMA_TO_DEVICE);
529	if (ret)
530	return ret;
531
532	dm_offset = CCP_SB_BYTES - aes->key_len;
533	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
534	if (ret)
535	goto e_key;
536	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
537	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
538	if (ret) {
539	cmd->engine_error = cmd_q->cmd_error;
540	goto e_key;
541	}
542
543	/* The AES context fits in a single (32-byte) SB entry and
544	* must be in little endian format. Use the 256-bit byte swap
545	* passthru option to convert from big endian to little endian.
546	*/
547	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
548	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
549	dir: DMA_BIDIRECTIONAL);
550	if (ret)
551	goto e_key;
552
553	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
554	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
555	if (ret)
556	goto e_ctx;
557	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
558	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
559	if (ret) {
560	cmd->engine_error = cmd_q->cmd_error;
561	goto e_ctx;
562	}
563
564	/* Send data to the CCP AES engine */
565	ret = ccp_init_data(data: &src, cmd_q, sg: aes->src, sg_len: aes->src_len,
566	AES_BLOCK_SIZE, dir: DMA_TO_DEVICE);
567	if (ret)
568	goto e_ctx;
569
570	while (src.sg_wa.bytes_left) {
571	ccp_prepare_data(src: &src, NULL, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
572	if (aes->cmac_final && !src.sg_wa.bytes_left) {
573	op.eom = 1;
574
575	/* Push the K1/K2 key to the CCP now */
576	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid,
577	sb: op.sb_ctx,
578	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
579	if (ret) {
580	cmd->engine_error = cmd_q->cmd_error;
581	goto e_src;
582	}
583
584	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: aes->cmac_key, sg_offset: 0,
585	len: aes->cmac_key_len);
586	if (ret)
587	goto e_src;
588	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
589	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
590	if (ret) {
591	cmd->engine_error = cmd_q->cmd_error;
592	goto e_src;
593	}
594	}
595
596	ret = cmd_q->ccp->vdata->perform->aes(&op);
597	if (ret) {
598	cmd->engine_error = cmd_q->cmd_error;
599	goto e_src;
600	}
601
602	ccp_process_data(src: &src, NULL, op: &op);
603	}
604
605	/* Retrieve the AES context - convert from LE to BE using
606	* 32-byte (256-bit) byteswapping
607	*/
608	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
609	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
610	if (ret) {
611	cmd->engine_error = cmd_q->cmd_error;
612	goto e_src;
613	}
614
615	/* ...but we only need AES_BLOCK_SIZE bytes */
616	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
617	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
618
619	e_src:
620	ccp_free_data(data: &src, cmd_q);
621
622	e_ctx:
623	ccp_dm_free(wa: &ctx);
624
625	e_key:
626	ccp_dm_free(wa: &key);
627
628	return ret;
629	}
630
631	static noinline_for_stack int
632	ccp_run_aes_gcm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
633	{
634	struct ccp_aes_engine *aes = &cmd->u.aes;
635	struct ccp_dm_workarea key, ctx, final_wa, tag;
636	struct ccp_data src, dst;
637	struct ccp_data aad;
638	struct ccp_op op;
639	unsigned int dm_offset;
640	unsigned int authsize;
641	unsigned int jobid;
642	unsigned int ilen;
643	bool in_place = true; /* Default value */
644	__be64 *final;
645	int ret;
646
647	struct scatterlist *p_inp, sg_inp[2];
648	struct scatterlist *p_tag, sg_tag[2];
649	struct scatterlist *p_outp, sg_outp[2];
650	struct scatterlist *p_aad;
651
652	if (!aes->iv)
653	return -EINVAL;
654
655	if (!((aes->key_len == AES_KEYSIZE_128) ||
656	(aes->key_len == AES_KEYSIZE_192) ||
657	(aes->key_len == AES_KEYSIZE_256)))
658	return -EINVAL;
659
660	if (!aes->key) /* Gotta have a key SGL */
661	return -EINVAL;
662
663	/* Zero defaults to 16 bytes, the maximum size */
664	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;
665	switch (authsize) {
666	case 16:
667	case 15:
668	case 14:
669	case 13:
670	case 12:
671	case 8:
672	case 4:
673	break;
674	default:
675	return -EINVAL;
676	}
677
678	/* First, decompose the source buffer into AAD & PT,
679	* and the destination buffer into AAD, CT & tag, or
680	* the input into CT & tag.
681	* It is expected that the input and output SGs will
682	* be valid, even if the AAD and input lengths are 0.
683	*/
684	p_aad = aes->src;
685	p_inp = scatterwalk_ffwd(dst: sg_inp, src: aes->src, len: aes->aad_len);
686	p_outp = scatterwalk_ffwd(dst: sg_outp, src: aes->dst, len: aes->aad_len);
687	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
688	ilen = aes->src_len;
689	p_tag = scatterwalk_ffwd(dst: sg_tag, src: p_outp, len: ilen);
690	} else {
691	/* Input length for decryption includes tag */
692	ilen = aes->src_len - authsize;
693	p_tag = scatterwalk_ffwd(dst: sg_tag, src: p_inp, len: ilen);
694	}
695
696	jobid = CCP_NEW_JOBID(cmd_q->ccp);
697
698	memset(&op, 0, sizeof(op));
699	op.cmd_q = cmd_q;
700	op.jobid = jobid;
701	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
702	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
703	op.init = 1;
704	op.u.aes.type = aes->type;
705
706	/* Copy the key to the LSB */
707	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
708	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
709	dir: DMA_TO_DEVICE);
710	if (ret)
711	return ret;
712
713	dm_offset = CCP_SB_BYTES - aes->key_len;
714	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
715	if (ret)
716	goto e_key;
717	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
718	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
719	if (ret) {
720	cmd->engine_error = cmd_q->cmd_error;
721	goto e_key;
722	}
723
724	/* Copy the context (IV) to the LSB.
725	* There is an assumption here that the IV is 96 bits in length, plus
726	* a nonce of 32 bits. If no IV is present, use a zeroed buffer.
727	*/
728	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
729	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
730	dir: DMA_BIDIRECTIONAL);
731	if (ret)
732	goto e_key;
733
734	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;
735	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
736	if (ret)
737	goto e_ctx;
738
739	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
740	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
741	if (ret) {
742	cmd->engine_error = cmd_q->cmd_error;
743	goto e_ctx;
744	}
745
746	op.init = 1;
747	if (aes->aad_len > 0) {
748	/* Step 1: Run a GHASH over the Additional Authenticated Data */
749	ret = ccp_init_data(data: &aad, cmd_q, sg: p_aad, sg_len: aes->aad_len,
750	AES_BLOCK_SIZE,
751	dir: DMA_TO_DEVICE);
752	if (ret)
753	goto e_ctx;
754
755	op.u.aes.mode = CCP_AES_MODE_GHASH;
756	op.u.aes.action = CCP_AES_GHASHAAD;
757
758	while (aad.sg_wa.bytes_left) {
759	ccp_prepare_data(src: &aad, NULL, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
760
761	ret = cmd_q->ccp->vdata->perform->aes(&op);
762	if (ret) {
763	cmd->engine_error = cmd_q->cmd_error;
764	goto e_aad;
765	}
766
767	ccp_process_data(src: &aad, NULL, op: &op);
768	op.init = 0;
769	}
770	}
771
772	op.u.aes.mode = CCP_AES_MODE_GCTR;
773	op.u.aes.action = aes->action;
774
775	if (ilen > 0) {
776	/* Step 2: Run a GCTR over the plaintext */
777	in_place = (sg_virt(sg: p_inp) == sg_virt(sg: p_outp)) ? true : false;
778
779	ret = ccp_init_data(data: &src, cmd_q, sg: p_inp, sg_len: ilen,
780	AES_BLOCK_SIZE,
781	dir: in_place ? DMA_BIDIRECTIONAL
782	: DMA_TO_DEVICE);
783	if (ret)
784	goto e_aad;
785
786	if (in_place) {
787	dst = src;
788	} else {
789	ret = ccp_init_data(data: &dst, cmd_q, sg: p_outp, sg_len: ilen,
790	AES_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
791	if (ret)
792	goto e_src;
793	}
794
795	op.soc = 0;
796	op.eom = 0;
797	op.init = 1;
798	while (src.sg_wa.bytes_left) {
799	ccp_prepare_data(src: &src, dst: &dst, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
800	if (!src.sg_wa.bytes_left) {
801	unsigned int nbytes = ilen % AES_BLOCK_SIZE;
802
803	if (nbytes) {
804	op.eom = 1;
805	op.u.aes.size = (nbytes * 8) - 1;
806	}
807	}
808
809	ret = cmd_q->ccp->vdata->perform->aes(&op);
810	if (ret) {
811	cmd->engine_error = cmd_q->cmd_error;
812	goto e_dst;
813	}
814
815	ccp_process_data(src: &src, dst: &dst, op: &op);
816	op.init = 0;
817	}
818	}
819
820	/* Step 3: Update the IV portion of the context with the original IV */
821	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
822	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
823	if (ret) {
824	cmd->engine_error = cmd_q->cmd_error;
825	goto e_dst;
826	}
827
828	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
829	if (ret)
830	goto e_dst;
831
832	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
833	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
834	if (ret) {
835	cmd->engine_error = cmd_q->cmd_error;
836	goto e_dst;
837	}
838
839	/* Step 4: Concatenate the lengths of the AAD and source, and
840	* hash that 16 byte buffer.
841	*/
842	ret = ccp_init_dm_workarea(wa: &final_wa, cmd_q, AES_BLOCK_SIZE,
843	dir: DMA_BIDIRECTIONAL);
844	if (ret)
845	goto e_dst;
846	final = (__be64 *)final_wa.address;
847	final[0] = cpu_to_be64(aes->aad_len * 8);
848	final[1] = cpu_to_be64(ilen * 8);
849
850	memset(&op, 0, sizeof(op));
851	op.cmd_q = cmd_q;
852	op.jobid = jobid;
853	op.sb_key = cmd_q->sb_key; /* Pre-allocated */
854	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
855	op.init = 1;
856	op.u.aes.type = aes->type;
857	op.u.aes.mode = CCP_AES_MODE_GHASH;
858	op.u.aes.action = CCP_AES_GHASHFINAL;
859	op.src.type = CCP_MEMTYPE_SYSTEM;
860	op.src.u.dma.address = final_wa.dma.address;
861	op.src.u.dma.length = AES_BLOCK_SIZE;
862	op.dst.type = CCP_MEMTYPE_SYSTEM;
863	op.dst.u.dma.address = final_wa.dma.address;
864	op.dst.u.dma.length = AES_BLOCK_SIZE;
865	op.eom = 1;
866	op.u.aes.size = 0;
867	ret = cmd_q->ccp->vdata->perform->aes(&op);
868	if (ret)
869	goto e_final_wa;
870
871	if (aes->action == CCP_AES_ACTION_ENCRYPT) {
872	/* Put the ciphered tag after the ciphertext. */
873	ccp_get_dm_area(wa: &final_wa, wa_offset: 0, sg: p_tag, sg_offset: 0, len: authsize);
874	} else {
875	/* Does this ciphered tag match the input? */
876	ret = ccp_init_dm_workarea(wa: &tag, cmd_q, len: authsize,
877	dir: DMA_BIDIRECTIONAL);
878	if (ret)
879	goto e_final_wa;
880	ret = ccp_set_dm_area(wa: &tag, wa_offset: 0, sg: p_tag, sg_offset: 0, len: authsize);
881	if (ret) {
882	ccp_dm_free(wa: &tag);
883	goto e_final_wa;
884	}
885
886	ret = crypto_memneq(a: tag.address, b: final_wa.address,
887	size: authsize) ? -EBADMSG : 0;
888	ccp_dm_free(wa: &tag);
889	}
890
891	e_final_wa:
892	ccp_dm_free(wa: &final_wa);
893
894	e_dst:
895	if (ilen > 0 && !in_place)
896	ccp_free_data(data: &dst, cmd_q);
897
898	e_src:
899	if (ilen > 0)
900	ccp_free_data(data: &src, cmd_q);
901
902	e_aad:
903	if (aes->aad_len)
904	ccp_free_data(data: &aad, cmd_q);
905
906	e_ctx:
907	ccp_dm_free(wa: &ctx);
908
909	e_key:
910	ccp_dm_free(wa: &key);
911
912	return ret;
913	}
914
915	static noinline_for_stack int
916	ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
917	{
918	struct ccp_aes_engine *aes = &cmd->u.aes;
919	struct ccp_dm_workarea key, ctx;
920	struct ccp_data src, dst;
921	struct ccp_op op;
922	unsigned int dm_offset;
923	bool in_place = false;
924	int ret;
925
926	if (!((aes->key_len == AES_KEYSIZE_128) ||
927	(aes->key_len == AES_KEYSIZE_192) ||
928	(aes->key_len == AES_KEYSIZE_256)))
929	return -EINVAL;
930
931	if (((aes->mode == CCP_AES_MODE_ECB) ||
932	(aes->mode == CCP_AES_MODE_CBC)) &&
933	(aes->src_len & (AES_BLOCK_SIZE - 1)))
934	return -EINVAL;
935
936	if (!aes->key || !aes->src || !aes->dst)
937	return -EINVAL;
938
939	if (aes->mode != CCP_AES_MODE_ECB) {
940	if (aes->iv_len != AES_BLOCK_SIZE)
941	return -EINVAL;
942
943	if (!aes->iv)
944	return -EINVAL;
945	}
946
947	BUILD_BUG_ON(CCP_AES_KEY_SB_COUNT != 1);
948	BUILD_BUG_ON(CCP_AES_CTX_SB_COUNT != 1);
949
950	ret = -EIO;
951	memset(&op, 0, sizeof(op));
952	op.cmd_q = cmd_q;
953	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
954	op.sb_key = cmd_q->sb_key;
955	op.sb_ctx = cmd_q->sb_ctx;
956	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;
957	op.u.aes.type = aes->type;
958	op.u.aes.mode = aes->mode;
959	op.u.aes.action = aes->action;
960
961	/* All supported key sizes fit in a single (32-byte) SB entry
962	* and must be in little endian format. Use the 256-bit byte
963	* swap passthru option to convert from big endian to little
964	* endian.
965	*/
966	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
967	CCP_AES_KEY_SB_COUNT * CCP_SB_BYTES,
968	dir: DMA_TO_DEVICE);
969	if (ret)
970	return ret;
971
972	dm_offset = CCP_SB_BYTES - aes->key_len;
973	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: aes->key, sg_offset: 0, len: aes->key_len);
974	if (ret)
975	goto e_key;
976	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
977	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
978	if (ret) {
979	cmd->engine_error = cmd_q->cmd_error;
980	goto e_key;
981	}
982
983	/* The AES context fits in a single (32-byte) SB entry and
984	* must be in little endian format. Use the 256-bit byte swap
985	* passthru option to convert from big endian to little endian.
986	*/
987	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
988	CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES,
989	dir: DMA_BIDIRECTIONAL);
990	if (ret)
991	goto e_key;
992
993	if (aes->mode != CCP_AES_MODE_ECB) {
994	/* Load the AES context - convert to LE */
995	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
996	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
997	if (ret)
998	goto e_ctx;
999	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1000	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1001	if (ret) {
1002	cmd->engine_error = cmd_q->cmd_error;
1003	goto e_ctx;
1004	}
1005	}
1006	switch (aes->mode) {
1007	case CCP_AES_MODE_CFB: /* CFB128 only */
1008	case CCP_AES_MODE_CTR:
1009	op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;
1010	break;
1011	default:
1012	op.u.aes.size = 0;
1013	}
1014
1015	/* Prepare the input and output data workareas. For in-place
1016	* operations we need to set the dma direction to BIDIRECTIONAL
1017	* and copy the src workarea to the dst workarea.
1018	*/
1019	if (sg_virt(sg: aes->src) == sg_virt(sg: aes->dst))
1020	in_place = true;
1021
1022	ret = ccp_init_data(data: &src, cmd_q, sg: aes->src, sg_len: aes->src_len,
1023	AES_BLOCK_SIZE,
1024	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1025	if (ret)
1026	goto e_ctx;
1027
1028	if (in_place) {
1029	dst = src;
1030	} else {
1031	ret = ccp_init_data(data: &dst, cmd_q, sg: aes->dst, sg_len: aes->src_len,
1032	AES_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
1033	if (ret)
1034	goto e_src;
1035	}
1036
1037	/* Send data to the CCP AES engine */
1038	while (src.sg_wa.bytes_left) {
1039	ccp_prepare_data(src: &src, dst: &dst, op: &op, AES_BLOCK_SIZE, blocksize_op: true);
1040	if (!src.sg_wa.bytes_left) {
1041	op.eom = 1;
1042
1043	/* Since we don't retrieve the AES context in ECB
1044	* mode we have to wait for the operation to complete
1045	* on the last piece of data
1046	*/
1047	if (aes->mode == CCP_AES_MODE_ECB)
1048	op.soc = 1;
1049	}
1050
1051	ret = cmd_q->ccp->vdata->perform->aes(&op);
1052	if (ret) {
1053	cmd->engine_error = cmd_q->cmd_error;
1054	goto e_dst;
1055	}
1056
1057	ccp_process_data(src: &src, dst: &dst, op: &op);
1058	}
1059
1060	if (aes->mode != CCP_AES_MODE_ECB) {
1061	/* Retrieve the AES context - convert from LE to BE using
1062	* 32-byte (256-bit) byteswapping
1063	*/
1064	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1065	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1066	if (ret) {
1067	cmd->engine_error = cmd_q->cmd_error;
1068	goto e_dst;
1069	}
1070
1071	/* ...but we only need AES_BLOCK_SIZE bytes */
1072	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1073	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: aes->iv, sg_offset: 0, len: aes->iv_len);
1074	}
1075
1076	e_dst:
1077	if (!in_place)
1078	ccp_free_data(data: &dst, cmd_q);
1079
1080	e_src:
1081	ccp_free_data(data: &src, cmd_q);
1082
1083	e_ctx:
1084	ccp_dm_free(wa: &ctx);
1085
1086	e_key:
1087	ccp_dm_free(wa: &key);
1088
1089	return ret;
1090	}
1091
1092	static noinline_for_stack int
1093	ccp_run_xts_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1094	{
1095	struct ccp_xts_aes_engine *xts = &cmd->u.xts;
1096	struct ccp_dm_workarea key, ctx;
1097	struct ccp_data src, dst;
1098	struct ccp_op op;
1099	unsigned int unit_size, dm_offset;
1100	bool in_place = false;
1101	unsigned int sb_count;
1102	enum ccp_aes_type aestype;
1103	int ret;
1104
1105	switch (xts->unit_size) {
1106	case CCP_XTS_AES_UNIT_SIZE_16:
1107	unit_size = 16;
1108	break;
1109	case CCP_XTS_AES_UNIT_SIZE_512:
1110	unit_size = 512;
1111	break;
1112	case CCP_XTS_AES_UNIT_SIZE_1024:
1113	unit_size = 1024;
1114	break;
1115	case CCP_XTS_AES_UNIT_SIZE_2048:
1116	unit_size = 2048;
1117	break;
1118	case CCP_XTS_AES_UNIT_SIZE_4096:
1119	unit_size = 4096;
1120	break;
1121
1122	default:
1123	return -EINVAL;
1124	}
1125
1126	if (xts->key_len == AES_KEYSIZE_128)
1127	aestype = CCP_AES_TYPE_128;
1128	else if (xts->key_len == AES_KEYSIZE_256)
1129	aestype = CCP_AES_TYPE_256;
1130	else
1131	return -EINVAL;
1132
1133	if (!xts->final && (xts->src_len & (AES_BLOCK_SIZE - 1)))
1134	return -EINVAL;
1135
1136	if (xts->iv_len != AES_BLOCK_SIZE)
1137	return -EINVAL;
1138
1139	if (!xts->key || !xts->iv || !xts->src || !xts->dst)
1140	return -EINVAL;
1141
1142	BUILD_BUG_ON(CCP_XTS_AES_KEY_SB_COUNT != 1);
1143	BUILD_BUG_ON(CCP_XTS_AES_CTX_SB_COUNT != 1);
1144
1145	ret = -EIO;
1146	memset(&op, 0, sizeof(op));
1147	op.cmd_q = cmd_q;
1148	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1149	op.sb_key = cmd_q->sb_key;
1150	op.sb_ctx = cmd_q->sb_ctx;
1151	op.init = 1;
1152	op.u.xts.type = aestype;
1153	op.u.xts.action = xts->action;
1154	op.u.xts.unit_size = xts->unit_size;
1155
1156	/* A version 3 device only supports 128-bit keys, which fits into a
1157	* single SB entry. A version 5 device uses a 512-bit vector, so two
1158	* SB entries.
1159	*/
1160	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0))
1161	sb_count = CCP_XTS_AES_KEY_SB_COUNT;
1162	else
1163	sb_count = CCP5_XTS_AES_KEY_SB_COUNT;
1164	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
1165	len: sb_count * CCP_SB_BYTES,
1166	dir: DMA_TO_DEVICE);
1167	if (ret)
1168	return ret;
1169
1170	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1171	/* All supported key sizes must be in little endian format.
1172	* Use the 256-bit byte swap passthru option to convert from
1173	* big endian to little endian.
1174	*/
1175	dm_offset = CCP_SB_BYTES - AES_KEYSIZE_128;
1176	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset, sg: xts->key, sg_offset: 0, len: xts->key_len);
1177	if (ret)
1178	goto e_key;
1179	ret = ccp_set_dm_area(wa: &key, wa_offset: 0, sg: xts->key, sg_offset: xts->key_len, len: xts->key_len);
1180	if (ret)
1181	goto e_key;
1182	} else {
1183	/* Version 5 CCPs use a 512-bit space for the key: each portion
1184	* occupies 256 bits, or one entire slot, and is zero-padded.
1185	*/
1186	unsigned int pad;
1187
1188	dm_offset = CCP_SB_BYTES;
1189	pad = dm_offset - xts->key_len;
1190	ret = ccp_set_dm_area(wa: &key, wa_offset: pad, sg: xts->key, sg_offset: 0, len: xts->key_len);
1191	if (ret)
1192	goto e_key;
1193	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + pad, sg: xts->key,
1194	sg_offset: xts->key_len, len: xts->key_len);
1195	if (ret)
1196	goto e_key;
1197	}
1198	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
1199	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1200	if (ret) {
1201	cmd->engine_error = cmd_q->cmd_error;
1202	goto e_key;
1203	}
1204
1205	/* The AES context fits in a single (32-byte) SB entry and
1206	* for XTS is already in little endian format so no byte swapping
1207	* is needed.
1208	*/
1209	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
1210	CCP_XTS_AES_CTX_SB_COUNT * CCP_SB_BYTES,
1211	dir: DMA_BIDIRECTIONAL);
1212	if (ret)
1213	goto e_key;
1214
1215	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: xts->iv, sg_offset: 0, len: xts->iv_len);
1216	if (ret)
1217	goto e_ctx;
1218	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1219	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
1220	if (ret) {
1221	cmd->engine_error = cmd_q->cmd_error;
1222	goto e_ctx;
1223	}
1224
1225	/* Prepare the input and output data workareas. For in-place
1226	* operations we need to set the dma direction to BIDIRECTIONAL
1227	* and copy the src workarea to the dst workarea.
1228	*/
1229	if (sg_virt(sg: xts->src) == sg_virt(sg: xts->dst))
1230	in_place = true;
1231
1232	ret = ccp_init_data(data: &src, cmd_q, sg: xts->src, sg_len: xts->src_len,
1233	dm_len: unit_size,
1234	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1235	if (ret)
1236	goto e_ctx;
1237
1238	if (in_place) {
1239	dst = src;
1240	} else {
1241	ret = ccp_init_data(data: &dst, cmd_q, sg: xts->dst, sg_len: xts->src_len,
1242	dm_len: unit_size, dir: DMA_FROM_DEVICE);
1243	if (ret)
1244	goto e_src;
1245	}
1246
1247	/* Send data to the CCP AES engine */
1248	while (src.sg_wa.bytes_left) {
1249	ccp_prepare_data(src: &src, dst: &dst, op: &op, block_size: unit_size, blocksize_op: true);
1250	if (!src.sg_wa.bytes_left)
1251	op.eom = 1;
1252
1253	ret = cmd_q->ccp->vdata->perform->xts_aes(&op);
1254	if (ret) {
1255	cmd->engine_error = cmd_q->cmd_error;
1256	goto e_dst;
1257	}
1258
1259	ccp_process_data(src: &src, dst: &dst, op: &op);
1260	}
1261
1262	/* Retrieve the AES context - convert from LE to BE using
1263	* 32-byte (256-bit) byteswapping
1264	*/
1265	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1266	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1267	if (ret) {
1268	cmd->engine_error = cmd_q->cmd_error;
1269	goto e_dst;
1270	}
1271
1272	/* ...but we only need AES_BLOCK_SIZE bytes */
1273	dm_offset = CCP_SB_BYTES - AES_BLOCK_SIZE;
1274	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: xts->iv, sg_offset: 0, len: xts->iv_len);
1275
1276	e_dst:
1277	if (!in_place)
1278	ccp_free_data(data: &dst, cmd_q);
1279
1280	e_src:
1281	ccp_free_data(data: &src, cmd_q);
1282
1283	e_ctx:
1284	ccp_dm_free(wa: &ctx);
1285
1286	e_key:
1287	ccp_dm_free(wa: &key);
1288
1289	return ret;
1290	}
1291
1292	static noinline_for_stack int
1293	ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1294	{
1295	struct ccp_des3_engine *des3 = &cmd->u.des3;
1296
1297	struct ccp_dm_workarea key, ctx;
1298	struct ccp_data src, dst;
1299	struct ccp_op op;
1300	unsigned int dm_offset;
1301	unsigned int len_singlekey;
1302	bool in_place = false;
1303	int ret;
1304
1305	/* Error checks */
1306	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0))
1307	return -EINVAL;
1308
1309	if (!cmd_q->ccp->vdata->perform->des3)
1310	return -EINVAL;
1311
1312	if (des3->key_len != DES3_EDE_KEY_SIZE)
1313	return -EINVAL;
1314
1315	if (((des3->mode == CCP_DES3_MODE_ECB) ||
1316	(des3->mode == CCP_DES3_MODE_CBC)) &&
1317	(des3->src_len & (DES3_EDE_BLOCK_SIZE - 1)))
1318	return -EINVAL;
1319
1320	if (!des3->key || !des3->src || !des3->dst)
1321	return -EINVAL;
1322
1323	if (des3->mode != CCP_DES3_MODE_ECB) {
1324	if (des3->iv_len != DES3_EDE_BLOCK_SIZE)
1325	return -EINVAL;
1326
1327	if (!des3->iv)
1328	return -EINVAL;
1329	}
1330
1331	/* Zero out all the fields of the command desc */
1332	memset(&op, 0, sizeof(op));
1333
1334	/* Set up the Function field */
1335	op.cmd_q = cmd_q;
1336	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1337	op.sb_key = cmd_q->sb_key;
1338
1339	op.init = (des3->mode == CCP_DES3_MODE_ECB) ? 0 : 1;
1340	op.u.des3.type = des3->type;
1341	op.u.des3.mode = des3->mode;
1342	op.u.des3.action = des3->action;
1343
1344	/*
1345	* All supported key sizes fit in a single (32-byte) KSB entry and
1346	* (like AES) must be in little endian format. Use the 256-bit byte
1347	* swap passthru option to convert from big endian to little endian.
1348	*/
1349	ret = ccp_init_dm_workarea(wa: &key, cmd_q,
1350	CCP_DES3_KEY_SB_COUNT * CCP_SB_BYTES,
1351	dir: DMA_TO_DEVICE);
1352	if (ret)
1353	return ret;
1354
1355	/*
1356	* The contents of the key triplet are in the reverse order of what
1357	* is required by the engine. Copy the 3 pieces individually to put
1358	* them where they belong.
1359	*/
1360	dm_offset = CCP_SB_BYTES - des3->key_len; /* Basic offset */
1361
1362	len_singlekey = des3->key_len / 3;
1363	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + 2 * len_singlekey,
1364	sg: des3->key, sg_offset: 0, len: len_singlekey);
1365	if (ret)
1366	goto e_key;
1367	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset + len_singlekey,
1368	sg: des3->key, sg_offset: len_singlekey, len: len_singlekey);
1369	if (ret)
1370	goto e_key;
1371	ret = ccp_set_dm_area(wa: &key, wa_offset: dm_offset,
1372	sg: des3->key, sg_offset: 2 * len_singlekey, len: len_singlekey);
1373	if (ret)
1374	goto e_key;
1375
1376	/* Copy the key to the SB */
1377	ret = ccp_copy_to_sb(cmd_q, wa: &key, jobid: op.jobid, sb: op.sb_key,
1378	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1379	if (ret) {
1380	cmd->engine_error = cmd_q->cmd_error;
1381	goto e_key;
1382	}
1383
1384	/*
1385	* The DES3 context fits in a single (32-byte) KSB entry and
1386	* must be in little endian format. Use the 256-bit byte swap
1387	* passthru option to convert from big endian to little endian.
1388	*/
1389	if (des3->mode != CCP_DES3_MODE_ECB) {
1390	op.sb_ctx = cmd_q->sb_ctx;
1391
1392	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q,
1393	CCP_DES3_CTX_SB_COUNT * CCP_SB_BYTES,
1394	dir: DMA_BIDIRECTIONAL);
1395	if (ret)
1396	goto e_key;
1397
1398	/* Load the context into the LSB */
1399	dm_offset = CCP_SB_BYTES - des3->iv_len;
1400	ret = ccp_set_dm_area(wa: &ctx, wa_offset: dm_offset, sg: des3->iv, sg_offset: 0,
1401	len: des3->iv_len);
1402	if (ret)
1403	goto e_ctx;
1404
1405	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1406	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1407	if (ret) {
1408	cmd->engine_error = cmd_q->cmd_error;
1409	goto e_ctx;
1410	}
1411	}
1412
1413	/*
1414	* Prepare the input and output data workareas. For in-place
1415	* operations we need to set the dma direction to BIDIRECTIONAL
1416	* and copy the src workarea to the dst workarea.
1417	*/
1418	if (sg_virt(sg: des3->src) == sg_virt(sg: des3->dst))
1419	in_place = true;
1420
1421	ret = ccp_init_data(data: &src, cmd_q, sg: des3->src, sg_len: des3->src_len,
1422	DES3_EDE_BLOCK_SIZE,
1423	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
1424	if (ret)
1425	goto e_ctx;
1426
1427	if (in_place)
1428	dst = src;
1429	else {
1430	ret = ccp_init_data(data: &dst, cmd_q, sg: des3->dst, sg_len: des3->src_len,
1431	DES3_EDE_BLOCK_SIZE, dir: DMA_FROM_DEVICE);
1432	if (ret)
1433	goto e_src;
1434	}
1435
1436	/* Send data to the CCP DES3 engine */
1437	while (src.sg_wa.bytes_left) {
1438	ccp_prepare_data(src: &src, dst: &dst, op: &op, DES3_EDE_BLOCK_SIZE, blocksize_op: true);
1439	if (!src.sg_wa.bytes_left) {
1440	op.eom = 1;
1441
1442	/* Since we don't retrieve the context in ECB mode
1443	* we have to wait for the operation to complete
1444	* on the last piece of data
1445	*/
1446	op.soc = 0;
1447	}
1448
1449	ret = cmd_q->ccp->vdata->perform->des3(&op);
1450	if (ret) {
1451	cmd->engine_error = cmd_q->cmd_error;
1452	goto e_dst;
1453	}
1454
1455	ccp_process_data(src: &src, dst: &dst, op: &op);
1456	}
1457
1458	if (des3->mode != CCP_DES3_MODE_ECB) {
1459	/* Retrieve the context and make BE */
1460	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1461	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1462	if (ret) {
1463	cmd->engine_error = cmd_q->cmd_error;
1464	goto e_dst;
1465	}
1466
1467	/* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */
1468	ccp_get_dm_area(wa: &ctx, wa_offset: dm_offset, sg: des3->iv, sg_offset: 0,
1469	DES3_EDE_BLOCK_SIZE);
1470	}
1471	e_dst:
1472	if (!in_place)
1473	ccp_free_data(data: &dst, cmd_q);
1474
1475	e_src:
1476	ccp_free_data(data: &src, cmd_q);
1477
1478	e_ctx:
1479	if (des3->mode != CCP_DES3_MODE_ECB)
1480	ccp_dm_free(wa: &ctx);
1481
1482	e_key:
1483	ccp_dm_free(wa: &key);
1484
1485	return ret;
1486	}
1487
1488	static noinline_for_stack int
1489	ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1490	{
1491	struct ccp_sha_engine *sha = &cmd->u.sha;
1492	struct ccp_dm_workarea ctx;
1493	struct ccp_data src;
1494	struct ccp_op op;
1495	unsigned int ioffset, ooffset;
1496	unsigned int digest_size;
1497	int sb_count;
1498	const void *init;
1499	u64 block_size;
1500	int ctx_size;
1501	int ret;
1502
1503	switch (sha->type) {
1504	case CCP_SHA_TYPE_1:
1505	if (sha->ctx_len < SHA1_DIGEST_SIZE)
1506	return -EINVAL;
1507	block_size = SHA1_BLOCK_SIZE;
1508	break;
1509	case CCP_SHA_TYPE_224:
1510	if (sha->ctx_len < SHA224_DIGEST_SIZE)
1511	return -EINVAL;
1512	block_size = SHA224_BLOCK_SIZE;
1513	break;
1514	case CCP_SHA_TYPE_256:
1515	if (sha->ctx_len < SHA256_DIGEST_SIZE)
1516	return -EINVAL;
1517	block_size = SHA256_BLOCK_SIZE;
1518	break;
1519	case CCP_SHA_TYPE_384:
1520	if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1521	|| sha->ctx_len < SHA384_DIGEST_SIZE)
1522	return -EINVAL;
1523	block_size = SHA384_BLOCK_SIZE;
1524	break;
1525	case CCP_SHA_TYPE_512:
1526	if (cmd_q->ccp->vdata->version < CCP_VERSION(4, 0)
1527	|| sha->ctx_len < SHA512_DIGEST_SIZE)
1528	return -EINVAL;
1529	block_size = SHA512_BLOCK_SIZE;
1530	break;
1531	default:
1532	return -EINVAL;
1533	}
1534
1535	if (!sha->ctx)
1536	return -EINVAL;
1537
1538	if (!sha->final && (sha->src_len & (block_size - 1)))
1539	return -EINVAL;
1540
1541	/* The version 3 device can't handle zero-length input */
1542	if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) {
1543
1544	if (!sha->src_len) {
1545	unsigned int digest_len;
1546	const u8 *sha_zero;
1547
1548	/* Not final, just return */
1549	if (!sha->final)
1550	return 0;
1551
1552	/* CCP can't do a zero length sha operation so the
1553	* caller must buffer the data.
1554	*/
1555	if (sha->msg_bits)
1556	return -EINVAL;
1557
1558	/* The CCP cannot perform zero-length sha operations
1559	* so the caller is required to buffer data for the
1560	* final operation. However, a sha operation for a
1561	* message with a total length of zero is valid so
1562	* known values are required to supply the result.
1563	*/
1564	switch (sha->type) {
1565	case CCP_SHA_TYPE_1:
1566	sha_zero = sha1_zero_message_hash;
1567	digest_len = SHA1_DIGEST_SIZE;
1568	break;
1569	case CCP_SHA_TYPE_224:
1570	sha_zero = sha224_zero_message_hash;
1571	digest_len = SHA224_DIGEST_SIZE;
1572	break;
1573	case CCP_SHA_TYPE_256:
1574	sha_zero = sha256_zero_message_hash;
1575	digest_len = SHA256_DIGEST_SIZE;
1576	break;
1577	default:
1578	return -EINVAL;
1579	}
1580
1581	scatterwalk_map_and_copy(buf: (void *)sha_zero, sg: sha->ctx, start: 0,
1582	nbytes: digest_len, out: 1);
1583
1584	return 0;
1585	}
1586	}
1587
1588	/* Set variables used throughout */
1589	switch (sha->type) {
1590	case CCP_SHA_TYPE_1:
1591	digest_size = SHA1_DIGEST_SIZE;
1592	init = (void *) ccp_sha1_init;
1593	ctx_size = SHA1_DIGEST_SIZE;
1594	sb_count = 1;
1595	if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1596	ooffset = ioffset = CCP_SB_BYTES - SHA1_DIGEST_SIZE;
1597	else
1598	ooffset = ioffset = 0;
1599	break;
1600	case CCP_SHA_TYPE_224:
1601	digest_size = SHA224_DIGEST_SIZE;
1602	init = (void *) ccp_sha224_init;
1603	ctx_size = SHA256_DIGEST_SIZE;
1604	sb_count = 1;
1605	ioffset = 0;
1606	if (cmd_q->ccp->vdata->version != CCP_VERSION(3, 0))
1607	ooffset = CCP_SB_BYTES - SHA224_DIGEST_SIZE;
1608	else
1609	ooffset = 0;
1610	break;
1611	case CCP_SHA_TYPE_256:
1612	digest_size = SHA256_DIGEST_SIZE;
1613	init = (void *) ccp_sha256_init;
1614	ctx_size = SHA256_DIGEST_SIZE;
1615	sb_count = 1;
1616	ooffset = ioffset = 0;
1617	break;
1618	case CCP_SHA_TYPE_384:
1619	digest_size = SHA384_DIGEST_SIZE;
1620	init = (void *) ccp_sha384_init;
1621	ctx_size = SHA512_DIGEST_SIZE;
1622	sb_count = 2;
1623	ioffset = 0;
1624	ooffset = 2 * CCP_SB_BYTES - SHA384_DIGEST_SIZE;
1625	break;
1626	case CCP_SHA_TYPE_512:
1627	digest_size = SHA512_DIGEST_SIZE;
1628	init = (void *) ccp_sha512_init;
1629	ctx_size = SHA512_DIGEST_SIZE;
1630	sb_count = 2;
1631	ooffset = ioffset = 0;
1632	break;
1633	default:
1634	ret = -EINVAL;
1635	goto e_data;
1636	}
1637
1638	/* For zero-length plaintext the src pointer is ignored;
1639	* otherwise both parts must be valid
1640	*/
1641	if (sha->src_len && !sha->src)
1642	return -EINVAL;
1643
1644	memset(&op, 0, sizeof(op));
1645	op.cmd_q = cmd_q;
1646	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1647	op.sb_ctx = cmd_q->sb_ctx; /* Pre-allocated */
1648	op.u.sha.type = sha->type;
1649	op.u.sha.msg_bits = sha->msg_bits;
1650
1651	/* For SHA1/224/256 the context fits in a single (32-byte) SB entry;
1652	* SHA384/512 require 2 adjacent SB slots, with the right half in the
1653	* first slot, and the left half in the second. Each portion must then
1654	* be in little endian format: use the 256-bit byte swap option.
1655	*/
1656	ret = ccp_init_dm_workarea(wa: &ctx, cmd_q, len: sb_count * CCP_SB_BYTES,
1657	dir: DMA_BIDIRECTIONAL);
1658	if (ret)
1659	return ret;
1660	if (sha->first) {
1661	switch (sha->type) {
1662	case CCP_SHA_TYPE_1:
1663	case CCP_SHA_TYPE_224:
1664	case CCP_SHA_TYPE_256:
1665	memcpy(ctx.address + ioffset, init, ctx_size);
1666	break;
1667	case CCP_SHA_TYPE_384:
1668	case CCP_SHA_TYPE_512:
1669	memcpy(ctx.address + ctx_size / 2, init,
1670	ctx_size / 2);
1671	memcpy(ctx.address, init + ctx_size / 2,
1672	ctx_size / 2);
1673	break;
1674	default:
1675	ret = -EINVAL;
1676	goto e_ctx;
1677	}
1678	} else {
1679	/* Restore the context */
1680	ret = ccp_set_dm_area(wa: &ctx, wa_offset: 0, sg: sha->ctx, sg_offset: 0,
1681	len: sb_count * CCP_SB_BYTES);
1682	if (ret)
1683	goto e_ctx;
1684	}
1685
1686	ret = ccp_copy_to_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1687	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1688	if (ret) {
1689	cmd->engine_error = cmd_q->cmd_error;
1690	goto e_ctx;
1691	}
1692
1693	if (sha->src) {
1694	/* Send data to the CCP SHA engine; block_size is set above */
1695	ret = ccp_init_data(data: &src, cmd_q, sg: sha->src, sg_len: sha->src_len,
1696	dm_len: block_size, dir: DMA_TO_DEVICE);
1697	if (ret)
1698	goto e_ctx;
1699
1700	while (src.sg_wa.bytes_left) {
1701	ccp_prepare_data(src: &src, NULL, op: &op, block_size, blocksize_op: false);
1702	if (sha->final && !src.sg_wa.bytes_left)
1703	op.eom = 1;
1704
1705	ret = cmd_q->ccp->vdata->perform->sha(&op);
1706	if (ret) {
1707	cmd->engine_error = cmd_q->cmd_error;
1708	goto e_data;
1709	}
1710
1711	ccp_process_data(src: &src, NULL, op: &op);
1712	}
1713	} else {
1714	op.eom = 1;
1715	ret = cmd_q->ccp->vdata->perform->sha(&op);
1716	if (ret) {
1717	cmd->engine_error = cmd_q->cmd_error;
1718	goto e_data;
1719	}
1720	}
1721
1722	/* Retrieve the SHA context - convert from LE to BE using
1723	* 32-byte (256-bit) byteswapping to BE
1724	*/
1725	ret = ccp_copy_from_sb(cmd_q, wa: &ctx, jobid: op.jobid, sb: op.sb_ctx,
1726	byte_swap: CCP_PASSTHRU_BYTESWAP_256BIT);
1727	if (ret) {
1728	cmd->engine_error = cmd_q->cmd_error;
1729	goto e_data;
1730	}
1731
1732	if (sha->final) {
1733	/* Finishing up, so get the digest */
1734	switch (sha->type) {
1735	case CCP_SHA_TYPE_1:
1736	case CCP_SHA_TYPE_224:
1737	case CCP_SHA_TYPE_256:
1738	ccp_get_dm_area(wa: &ctx, wa_offset: ooffset,
1739	sg: sha->ctx, sg_offset: 0,
1740	len: digest_size);
1741	break;
1742	case CCP_SHA_TYPE_384:
1743	case CCP_SHA_TYPE_512:
1744	ccp_get_dm_area(wa: &ctx, wa_offset: 0,
1745	sg: sha->ctx, LSB_ITEM_SIZE - ooffset,
1746	LSB_ITEM_SIZE);
1747	ccp_get_dm_area(wa: &ctx, LSB_ITEM_SIZE + ooffset,
1748	sg: sha->ctx, sg_offset: 0,
1749	LSB_ITEM_SIZE - ooffset);
1750	break;
1751	default:
1752	ret = -EINVAL;
1753	goto e_data;
1754	}
1755	} else {
1756	/* Stash the context */
1757	ccp_get_dm_area(wa: &ctx, wa_offset: 0, sg: sha->ctx, sg_offset: 0,
1758	len: sb_count * CCP_SB_BYTES);
1759	}
1760
1761	if (sha->final && sha->opad) {
1762	/* HMAC operation, recursively perform final SHA */
1763	struct ccp_cmd hmac_cmd;
1764	struct scatterlist sg;
1765	u8 *hmac_buf;
1766
1767	if (sha->opad_len != block_size) {
1768	ret = -EINVAL;
1769	goto e_data;
1770	}
1771
1772	hmac_buf = kmalloc(size: block_size + digest_size, GFP_KERNEL);
1773	if (!hmac_buf) {
1774	ret = -ENOMEM;
1775	goto e_data;
1776	}
1777	sg_init_one(&sg, hmac_buf, block_size + digest_size);
1778
1779	scatterwalk_map_and_copy(buf: hmac_buf, sg: sha->opad, start: 0, nbytes: block_size, out: 0);
1780	switch (sha->type) {
1781	case CCP_SHA_TYPE_1:
1782	case CCP_SHA_TYPE_224:
1783	case CCP_SHA_TYPE_256:
1784	memcpy(hmac_buf + block_size,
1785	ctx.address + ooffset,
1786	digest_size);
1787	break;
1788	case CCP_SHA_TYPE_384:
1789	case CCP_SHA_TYPE_512:
1790	memcpy(hmac_buf + block_size,
1791	ctx.address + LSB_ITEM_SIZE + ooffset,
1792	LSB_ITEM_SIZE);
1793	memcpy(hmac_buf + block_size +
1794	(LSB_ITEM_SIZE - ooffset),
1795	ctx.address,
1796	LSB_ITEM_SIZE);
1797	break;
1798	default:
1799	kfree(objp: hmac_buf);
1800	ret = -EINVAL;
1801	goto e_data;
1802	}
1803
1804	memset(&hmac_cmd, 0, sizeof(hmac_cmd));
1805	hmac_cmd.engine = CCP_ENGINE_SHA;
1806	hmac_cmd.u.sha.type = sha->type;
1807	hmac_cmd.u.sha.ctx = sha->ctx;
1808	hmac_cmd.u.sha.ctx_len = sha->ctx_len;
1809	hmac_cmd.u.sha.src = &sg;
1810	hmac_cmd.u.sha.src_len = block_size + digest_size;
1811	hmac_cmd.u.sha.opad = NULL;
1812	hmac_cmd.u.sha.opad_len = 0;
1813	hmac_cmd.u.sha.first = 1;
1814	hmac_cmd.u.sha.final = 1;
1815	hmac_cmd.u.sha.msg_bits = (block_size + digest_size) << 3;
1816
1817	ret = ccp_run_sha_cmd(cmd_q, cmd: &hmac_cmd);
1818	if (ret)
1819	cmd->engine_error = hmac_cmd.engine_error;
1820
1821	kfree(objp: hmac_buf);
1822	}
1823
1824	e_data:
1825	if (sha->src)
1826	ccp_free_data(data: &src, cmd_q);
1827
1828	e_ctx:
1829	ccp_dm_free(wa: &ctx);
1830
1831	return ret;
1832	}
1833
1834	static noinline_for_stack int
1835	ccp_run_rsa_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1836	{
1837	struct ccp_rsa_engine *rsa = &cmd->u.rsa;
1838	struct ccp_dm_workarea exp, src, dst;
1839	struct ccp_op op;
1840	unsigned int sb_count, i_len, o_len;
1841	int ret;
1842
1843	/* Check against the maximum allowable size, in bits */
1844	if (rsa->key_size > cmd_q->ccp->vdata->rsamax)
1845	return -EINVAL;
1846
1847	if (!rsa->exp || !rsa->mod || !rsa->src || !rsa->dst)
1848	return -EINVAL;
1849
1850	memset(&op, 0, sizeof(op));
1851	op.cmd_q = cmd_q;
1852	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1853
1854	/* The RSA modulus must precede the message being acted upon, so
1855	* it must be copied to a DMA area where the message and the
1856	* modulus can be concatenated. Therefore the input buffer
1857	* length required is twice the output buffer length (which
1858	* must be a multiple of 256-bits). Compute o_len, i_len in bytes.
1859	* Buffer sizes must be a multiple of 32 bytes; rounding up may be
1860	* required.
1861	*/
1862	o_len = 32 * ((rsa->key_size + 255) / 256);
1863	i_len = o_len * 2;
1864
1865	sb_count = 0;
1866	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1867	/* sb_count is the number of storage block slots required
1868	* for the modulus.
1869	*/
1870	sb_count = o_len / CCP_SB_BYTES;
1871	op.sb_key = cmd_q->ccp->vdata->perform->sballoc(cmd_q,
1872	sb_count);
1873	if (!op.sb_key)
1874	return -EIO;
1875	} else {
1876	/* A version 5 device allows a modulus size that will not fit
1877	* in the LSB, so the command will transfer it from memory.
1878	* Set the sb key to the default, even though it's not used.
1879	*/
1880	op.sb_key = cmd_q->sb_key;
1881	}
1882
1883	/* The RSA exponent must be in little endian format. Reverse its
1884	* byte order.
1885	*/
1886	ret = ccp_init_dm_workarea(wa: &exp, cmd_q, len: o_len, dir: DMA_TO_DEVICE);
1887	if (ret)
1888	goto e_sb;
1889
1890	ret = ccp_reverse_set_dm_area(wa: &exp, wa_offset: 0, sg: rsa->exp, sg_offset: 0, len: rsa->exp_len);
1891	if (ret)
1892	goto e_exp;
1893
1894	if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) {
1895	/* Copy the exponent to the local storage block, using
1896	* as many 32-byte blocks as were allocated above. It's
1897	* already little endian, so no further change is required.
1898	*/
1899	ret = ccp_copy_to_sb(cmd_q, wa: &exp, jobid: op.jobid, sb: op.sb_key,
1900	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
1901	if (ret) {
1902	cmd->engine_error = cmd_q->cmd_error;
1903	goto e_exp;
1904	}
1905	} else {
1906	/* The exponent can be retrieved from memory via DMA. */
1907	op.exp.u.dma.address = exp.dma.address;
1908	op.exp.u.dma.offset = 0;
1909	}
1910
1911	/* Concatenate the modulus and the message. Both the modulus and
1912	* the operands must be in little endian format. Since the input
1913	* is in big endian format it must be converted.
1914	*/
1915	ret = ccp_init_dm_workarea(wa: &src, cmd_q, len: i_len, dir: DMA_TO_DEVICE);
1916	if (ret)
1917	goto e_exp;
1918
1919	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: rsa->mod, sg_offset: 0, len: rsa->mod_len);
1920	if (ret)
1921	goto e_src;
1922	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: o_len, sg: rsa->src, sg_offset: 0, len: rsa->src_len);
1923	if (ret)
1924	goto e_src;
1925
1926	/* Prepare the output area for the operation */
1927	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, len: o_len, dir: DMA_FROM_DEVICE);
1928	if (ret)
1929	goto e_src;
1930
1931	op.soc = 1;
1932	op.src.u.dma.address = src.dma.address;
1933	op.src.u.dma.offset = 0;
1934	op.src.u.dma.length = i_len;
1935	op.dst.u.dma.address = dst.dma.address;
1936	op.dst.u.dma.offset = 0;
1937	op.dst.u.dma.length = o_len;
1938
1939	op.u.rsa.mod_size = rsa->key_size;
1940	op.u.rsa.input_len = i_len;
1941
1942	ret = cmd_q->ccp->vdata->perform->rsa(&op);
1943	if (ret) {
1944	cmd->engine_error = cmd_q->cmd_error;
1945	goto e_dst;
1946	}
1947
1948	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: rsa->dst, sg_offset: 0, len: rsa->mod_len);
1949
1950	e_dst:
1951	ccp_dm_free(wa: &dst);
1952
1953	e_src:
1954	ccp_dm_free(wa: &src);
1955
1956	e_exp:
1957	ccp_dm_free(wa: &exp);
1958
1959	e_sb:
1960	if (sb_count)
1961	cmd_q->ccp->vdata->perform->sbfree(cmd_q, op.sb_key, sb_count);
1962
1963	return ret;
1964	}
1965
1966	static noinline_for_stack int
1967	ccp_run_passthru_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
1968	{
1969	struct ccp_passthru_engine *pt = &cmd->u.passthru;
1970	struct ccp_dm_workarea mask;
1971	struct ccp_data src, dst;
1972	struct ccp_op op;
1973	bool in_place = false;
1974	unsigned int i;
1975	int ret = 0;
1976
1977	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
1978	return -EINVAL;
1979
1980	if (!pt->src || !pt->dst)
1981	return -EINVAL;
1982
1983	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1984	if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
1985	return -EINVAL;
1986	if (!pt->mask)
1987	return -EINVAL;
1988	}
1989
1990	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
1991
1992	memset(&op, 0, sizeof(op));
1993	op.cmd_q = cmd_q;
1994	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
1995
1996	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
1997	/* Load the mask */
1998	op.sb_key = cmd_q->sb_key;
1999
2000	ret = ccp_init_dm_workarea(wa: &mask, cmd_q,
2001	CCP_PASSTHRU_SB_COUNT *
2002	CCP_SB_BYTES,
2003	dir: DMA_TO_DEVICE);
2004	if (ret)
2005	return ret;
2006
2007	ret = ccp_set_dm_area(wa: &mask, wa_offset: 0, sg: pt->mask, sg_offset: 0, len: pt->mask_len);
2008	if (ret)
2009	goto e_mask;
2010	ret = ccp_copy_to_sb(cmd_q, wa: &mask, jobid: op.jobid, sb: op.sb_key,
2011	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
2012	if (ret) {
2013	cmd->engine_error = cmd_q->cmd_error;
2014	goto e_mask;
2015	}
2016	}
2017
2018	/* Prepare the input and output data workareas. For in-place
2019	* operations we need to set the dma direction to BIDIRECTIONAL
2020	* and copy the src workarea to the dst workarea.
2021	*/
2022	if (sg_virt(sg: pt->src) == sg_virt(sg: pt->dst))
2023	in_place = true;
2024
2025	ret = ccp_init_data(data: &src, cmd_q, sg: pt->src, sg_len: pt->src_len,
2026	CCP_PASSTHRU_MASKSIZE,
2027	dir: in_place ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE);
2028	if (ret)
2029	goto e_mask;
2030
2031	if (in_place) {
2032	dst = src;
2033	} else {
2034	ret = ccp_init_data(data: &dst, cmd_q, sg: pt->dst, sg_len: pt->src_len,
2035	CCP_PASSTHRU_MASKSIZE, dir: DMA_FROM_DEVICE);
2036	if (ret)
2037	goto e_src;
2038	}
2039
2040	/* Send data to the CCP Passthru engine
2041	* Because the CCP engine works on a single source and destination
2042	* dma address at a time, each entry in the source scatterlist
2043	* (after the dma_map_sg call) must be less than or equal to the
2044	* (remaining) length in the destination scatterlist entry and the
2045	* length must be a multiple of CCP_PASSTHRU_BLOCKSIZE
2046	*/
2047	dst.sg_wa.sg_used = 0;
2048	for (i = 1; i <= src.sg_wa.dma_count; i++) {
2049	if (!dst.sg_wa.sg ||
2050	(sg_dma_len(dst.sg_wa.sg) < sg_dma_len(src.sg_wa.sg))) {
2051	ret = -EINVAL;
2052	goto e_dst;
2053	}
2054
2055	if (i == src.sg_wa.dma_count) {
2056	op.eom = 1;
2057	op.soc = 1;
2058	}
2059
2060	op.src.type = CCP_MEMTYPE_SYSTEM;
2061	op.src.u.dma.address = sg_dma_address(src.sg_wa.sg);
2062	op.src.u.dma.offset = 0;
2063	op.src.u.dma.length = sg_dma_len(src.sg_wa.sg);
2064
2065	op.dst.type = CCP_MEMTYPE_SYSTEM;
2066	op.dst.u.dma.address = sg_dma_address(dst.sg_wa.sg);
2067	op.dst.u.dma.offset = dst.sg_wa.sg_used;
2068	op.dst.u.dma.length = op.src.u.dma.length;
2069
2070	ret = cmd_q->ccp->vdata->perform->passthru(&op);
2071	if (ret) {
2072	cmd->engine_error = cmd_q->cmd_error;
2073	goto e_dst;
2074	}
2075
2076	dst.sg_wa.sg_used += sg_dma_len(src.sg_wa.sg);
2077	if (dst.sg_wa.sg_used == sg_dma_len(dst.sg_wa.sg)) {
2078	dst.sg_wa.sg = sg_next(dst.sg_wa.sg);
2079	dst.sg_wa.sg_used = 0;
2080	}
2081	src.sg_wa.sg = sg_next(src.sg_wa.sg);
2082	}
2083
2084	e_dst:
2085	if (!in_place)
2086	ccp_free_data(data: &dst, cmd_q);
2087
2088	e_src:
2089	ccp_free_data(data: &src, cmd_q);
2090
2091	e_mask:
2092	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
2093	ccp_dm_free(wa: &mask);
2094
2095	return ret;
2096	}
2097
2098	static noinline_for_stack int
2099	ccp_run_passthru_nomap_cmd(struct ccp_cmd_queue *cmd_q,
2100	struct ccp_cmd *cmd)
2101	{
2102	struct ccp_passthru_nomap_engine *pt = &cmd->u.passthru_nomap;
2103	struct ccp_dm_workarea mask;
2104	struct ccp_op op;
2105	int ret;
2106
2107	if (!pt->final && (pt->src_len & (CCP_PASSTHRU_BLOCKSIZE - 1)))
2108	return -EINVAL;
2109
2110	if (!pt->src_dma || !pt->dst_dma)
2111	return -EINVAL;
2112
2113	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2114	if (pt->mask_len != CCP_PASSTHRU_MASKSIZE)
2115	return -EINVAL;
2116	if (!pt->mask)
2117	return -EINVAL;
2118	}
2119
2120	BUILD_BUG_ON(CCP_PASSTHRU_SB_COUNT != 1);
2121
2122	memset(&op, 0, sizeof(op));
2123	op.cmd_q = cmd_q;
2124	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2125
2126	if (pt->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) {
2127	/* Load the mask */
2128	op.sb_key = cmd_q->sb_key;
2129
2130	mask.length = pt->mask_len;
2131	mask.dma.address = pt->mask;
2132	mask.dma.length = pt->mask_len;
2133
2134	ret = ccp_copy_to_sb(cmd_q, wa: &mask, jobid: op.jobid, sb: op.sb_key,
2135	byte_swap: CCP_PASSTHRU_BYTESWAP_NOOP);
2136	if (ret) {
2137	cmd->engine_error = cmd_q->cmd_error;
2138	return ret;
2139	}
2140	}
2141
2142	/* Send data to the CCP Passthru engine */
2143	op.eom = 1;
2144	op.soc = 1;
2145
2146	op.src.type = CCP_MEMTYPE_SYSTEM;
2147	op.src.u.dma.address = pt->src_dma;
2148	op.src.u.dma.offset = 0;
2149	op.src.u.dma.length = pt->src_len;
2150
2151	op.dst.type = CCP_MEMTYPE_SYSTEM;
2152	op.dst.u.dma.address = pt->dst_dma;
2153	op.dst.u.dma.offset = 0;
2154	op.dst.u.dma.length = pt->src_len;
2155
2156	ret = cmd_q->ccp->vdata->perform->passthru(&op);
2157	if (ret)
2158	cmd->engine_error = cmd_q->cmd_error;
2159
2160	return ret;
2161	}
2162
2163	static int ccp_run_ecc_mm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2164	{
2165	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2166	struct ccp_dm_workarea src, dst;
2167	struct ccp_op op;
2168	int ret;
2169	u8 *save;
2170
2171	if (!ecc->u.mm.operand_1 ||
2172	(ecc->u.mm.operand_1_len > CCP_ECC_MODULUS_BYTES))
2173	return -EINVAL;
2174
2175	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT)
2176	if (!ecc->u.mm.operand_2 ||
2177	(ecc->u.mm.operand_2_len > CCP_ECC_MODULUS_BYTES))
2178	return -EINVAL;
2179
2180	if (!ecc->u.mm.result ||
2181	(ecc->u.mm.result_len < CCP_ECC_MODULUS_BYTES))
2182	return -EINVAL;
2183
2184	memset(&op, 0, sizeof(op));
2185	op.cmd_q = cmd_q;
2186	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2187
2188	/* Concatenate the modulus and the operands. Both the modulus and
2189	* the operands must be in little endian format. Since the input
2190	* is in big endian format it must be converted and placed in a
2191	* fixed length buffer.
2192	*/
2193	ret = ccp_init_dm_workarea(wa: &src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2194	dir: DMA_TO_DEVICE);
2195	if (ret)
2196	return ret;
2197
2198	/* Save the workarea address since it is updated in order to perform
2199	* the concatenation
2200	*/
2201	save = src.address;
2202
2203	/* Copy the ECC modulus */
2204	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->mod, sg_offset: 0, len: ecc->mod_len);
2205	if (ret)
2206	goto e_src;
2207	src.address += CCP_ECC_OPERAND_SIZE;
2208
2209	/* Copy the first operand */
2210	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.mm.operand_1, sg_offset: 0,
2211	len: ecc->u.mm.operand_1_len);
2212	if (ret)
2213	goto e_src;
2214	src.address += CCP_ECC_OPERAND_SIZE;
2215
2216	if (ecc->function != CCP_ECC_FUNCTION_MINV_384BIT) {
2217	/* Copy the second operand */
2218	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.mm.operand_2, sg_offset: 0,
2219	len: ecc->u.mm.operand_2_len);
2220	if (ret)
2221	goto e_src;
2222	src.address += CCP_ECC_OPERAND_SIZE;
2223	}
2224
2225	/* Restore the workarea address */
2226	src.address = save;
2227
2228	/* Prepare the output area for the operation */
2229	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2230	dir: DMA_FROM_DEVICE);
2231	if (ret)
2232	goto e_src;
2233
2234	op.soc = 1;
2235	op.src.u.dma.address = src.dma.address;
2236	op.src.u.dma.offset = 0;
2237	op.src.u.dma.length = src.length;
2238	op.dst.u.dma.address = dst.dma.address;
2239	op.dst.u.dma.offset = 0;
2240	op.dst.u.dma.length = dst.length;
2241
2242	op.u.ecc.function = cmd->u.ecc.function;
2243
2244	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2245	if (ret) {
2246	cmd->engine_error = cmd_q->cmd_error;
2247	goto e_dst;
2248	}
2249
2250	ecc->ecc_result = le16_to_cpup(
2251	p: (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2252	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2253	ret = -EIO;
2254	goto e_dst;
2255	}
2256
2257	/* Save the ECC result */
2258	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.mm.result, sg_offset: 0,
2259	CCP_ECC_MODULUS_BYTES);
2260
2261	e_dst:
2262	ccp_dm_free(wa: &dst);
2263
2264	e_src:
2265	ccp_dm_free(wa: &src);
2266
2267	return ret;
2268	}
2269
2270	static int ccp_run_ecc_pm_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2271	{
2272	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2273	struct ccp_dm_workarea src, dst;
2274	struct ccp_op op;
2275	int ret;
2276	u8 *save;
2277
2278	if (!ecc->u.pm.point_1.x ||
2279	(ecc->u.pm.point_1.x_len > CCP_ECC_MODULUS_BYTES) ||
2280	!ecc->u.pm.point_1.y ||
2281	(ecc->u.pm.point_1.y_len > CCP_ECC_MODULUS_BYTES))
2282	return -EINVAL;
2283
2284	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2285	if (!ecc->u.pm.point_2.x ||
2286	(ecc->u.pm.point_2.x_len > CCP_ECC_MODULUS_BYTES) ||
2287	!ecc->u.pm.point_2.y ||
2288	(ecc->u.pm.point_2.y_len > CCP_ECC_MODULUS_BYTES))
2289	return -EINVAL;
2290	} else {
2291	if (!ecc->u.pm.domain_a ||
2292	(ecc->u.pm.domain_a_len > CCP_ECC_MODULUS_BYTES))
2293	return -EINVAL;
2294
2295	if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT)
2296	if (!ecc->u.pm.scalar ||
2297	(ecc->u.pm.scalar_len > CCP_ECC_MODULUS_BYTES))
2298	return -EINVAL;
2299	}
2300
2301	if (!ecc->u.pm.result.x ||
2302	(ecc->u.pm.result.x_len < CCP_ECC_MODULUS_BYTES) ||
2303	!ecc->u.pm.result.y ||
2304	(ecc->u.pm.result.y_len < CCP_ECC_MODULUS_BYTES))
2305	return -EINVAL;
2306
2307	memset(&op, 0, sizeof(op));
2308	op.cmd_q = cmd_q;
2309	op.jobid = CCP_NEW_JOBID(cmd_q->ccp);
2310
2311	/* Concatenate the modulus and the operands. Both the modulus and
2312	* the operands must be in little endian format. Since the input
2313	* is in big endian format it must be converted and placed in a
2314	* fixed length buffer.
2315	*/
2316	ret = ccp_init_dm_workarea(wa: &src, cmd_q, CCP_ECC_SRC_BUF_SIZE,
2317	dir: DMA_TO_DEVICE);
2318	if (ret)
2319	return ret;
2320
2321	/* Save the workarea address since it is updated in order to perform
2322	* the concatenation
2323	*/
2324	save = src.address;
2325
2326	/* Copy the ECC modulus */
2327	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->mod, sg_offset: 0, len: ecc->mod_len);
2328	if (ret)
2329	goto e_src;
2330	src.address += CCP_ECC_OPERAND_SIZE;
2331
2332	/* Copy the first point X and Y coordinate */
2333	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_1.x, sg_offset: 0,
2334	len: ecc->u.pm.point_1.x_len);
2335	if (ret)
2336	goto e_src;
2337	src.address += CCP_ECC_OPERAND_SIZE;
2338	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_1.y, sg_offset: 0,
2339	len: ecc->u.pm.point_1.y_len);
2340	if (ret)
2341	goto e_src;
2342	src.address += CCP_ECC_OPERAND_SIZE;
2343
2344	/* Set the first point Z coordinate to 1 */
2345	*src.address = 0x01;
2346	src.address += CCP_ECC_OPERAND_SIZE;
2347
2348	if (ecc->function == CCP_ECC_FUNCTION_PADD_384BIT) {
2349	/* Copy the second point X and Y coordinate */
2350	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_2.x, sg_offset: 0,
2351	len: ecc->u.pm.point_2.x_len);
2352	if (ret)
2353	goto e_src;
2354	src.address += CCP_ECC_OPERAND_SIZE;
2355	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.point_2.y, sg_offset: 0,
2356	len: ecc->u.pm.point_2.y_len);
2357	if (ret)
2358	goto e_src;
2359	src.address += CCP_ECC_OPERAND_SIZE;
2360
2361	/* Set the second point Z coordinate to 1 */
2362	*src.address = 0x01;
2363	src.address += CCP_ECC_OPERAND_SIZE;
2364	} else {
2365	/* Copy the Domain "a" parameter */
2366	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0, sg: ecc->u.pm.domain_a, sg_offset: 0,
2367	len: ecc->u.pm.domain_a_len);
2368	if (ret)
2369	goto e_src;
2370	src.address += CCP_ECC_OPERAND_SIZE;
2371
2372	if (ecc->function == CCP_ECC_FUNCTION_PMUL_384BIT) {
2373	/* Copy the scalar value */
2374	ret = ccp_reverse_set_dm_area(wa: &src, wa_offset: 0,
2375	sg: ecc->u.pm.scalar, sg_offset: 0,
2376	len: ecc->u.pm.scalar_len);
2377	if (ret)
2378	goto e_src;
2379	src.address += CCP_ECC_OPERAND_SIZE;
2380	}
2381	}
2382
2383	/* Restore the workarea address */
2384	src.address = save;
2385
2386	/* Prepare the output area for the operation */
2387	ret = ccp_init_dm_workarea(wa: &dst, cmd_q, CCP_ECC_DST_BUF_SIZE,
2388	dir: DMA_FROM_DEVICE);
2389	if (ret)
2390	goto e_src;
2391
2392	op.soc = 1;
2393	op.src.u.dma.address = src.dma.address;
2394	op.src.u.dma.offset = 0;
2395	op.src.u.dma.length = src.length;
2396	op.dst.u.dma.address = dst.dma.address;
2397	op.dst.u.dma.offset = 0;
2398	op.dst.u.dma.length = dst.length;
2399
2400	op.u.ecc.function = cmd->u.ecc.function;
2401
2402	ret = cmd_q->ccp->vdata->perform->ecc(&op);
2403	if (ret) {
2404	cmd->engine_error = cmd_q->cmd_error;
2405	goto e_dst;
2406	}
2407
2408	ecc->ecc_result = le16_to_cpup(
2409	p: (const __le16 *)(dst.address + CCP_ECC_RESULT_OFFSET));
2410	if (!(ecc->ecc_result & CCP_ECC_RESULT_SUCCESS)) {
2411	ret = -EIO;
2412	goto e_dst;
2413	}
2414
2415	/* Save the workarea address since it is updated as we walk through
2416	* to copy the point math result
2417	*/
2418	save = dst.address;
2419
2420	/* Save the ECC result X and Y coordinates */
2421	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.pm.result.x, sg_offset: 0,
2422	CCP_ECC_MODULUS_BYTES);
2423	dst.address += CCP_ECC_OUTPUT_SIZE;
2424	ccp_reverse_get_dm_area(wa: &dst, wa_offset: 0, sg: ecc->u.pm.result.y, sg_offset: 0,
2425	CCP_ECC_MODULUS_BYTES);
2426
2427	/* Restore the workarea address */
2428	dst.address = save;
2429
2430	e_dst:
2431	ccp_dm_free(wa: &dst);
2432
2433	e_src:
2434	ccp_dm_free(wa: &src);
2435
2436	return ret;
2437	}
2438
2439	static noinline_for_stack int
2440	ccp_run_ecc_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2441	{
2442	struct ccp_ecc_engine *ecc = &cmd->u.ecc;
2443
2444	ecc->ecc_result = 0;
2445
2446	if (!ecc->mod ||
2447	(ecc->mod_len > CCP_ECC_MODULUS_BYTES))
2448	return -EINVAL;
2449
2450	switch (ecc->function) {
2451	case CCP_ECC_FUNCTION_MMUL_384BIT:
2452	case CCP_ECC_FUNCTION_MADD_384BIT:
2453	case CCP_ECC_FUNCTION_MINV_384BIT:
2454	return ccp_run_ecc_mm_cmd(cmd_q, cmd);
2455
2456	case CCP_ECC_FUNCTION_PADD_384BIT:
2457	case CCP_ECC_FUNCTION_PMUL_384BIT:
2458	case CCP_ECC_FUNCTION_PDBL_384BIT:
2459	return ccp_run_ecc_pm_cmd(cmd_q, cmd);
2460
2461	default:
2462	return -EINVAL;
2463	}
2464	}
2465
2466	int ccp_run_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd)
2467	{
2468	int ret;
2469
2470	cmd->engine_error = 0;
2471	cmd_q->cmd_error = 0;
2472	cmd_q->int_rcvd = 0;
2473	cmd_q->free_slots = cmd_q->ccp->vdata->perform->get_free_slots(cmd_q);
2474
2475	switch (cmd->engine) {
2476	case CCP_ENGINE_AES:
2477	switch (cmd->u.aes.mode) {
2478	case CCP_AES_MODE_CMAC:
2479	ret = ccp_run_aes_cmac_cmd(cmd_q, cmd);
2480	break;
2481	case CCP_AES_MODE_GCM:
2482	ret = ccp_run_aes_gcm_cmd(cmd_q, cmd);
2483	break;
2484	default:
2485	ret = ccp_run_aes_cmd(cmd_q, cmd);
2486	break;
2487	}
2488	break;
2489	case CCP_ENGINE_XTS_AES_128:
2490	ret = ccp_run_xts_aes_cmd(cmd_q, cmd);
2491	break;
2492	case CCP_ENGINE_DES3:
2493	ret = ccp_run_des3_cmd(cmd_q, cmd);
2494	break;
2495	case CCP_ENGINE_SHA:
2496	ret = ccp_run_sha_cmd(cmd_q, cmd);
2497	break;
2498	case CCP_ENGINE_RSA:
2499	ret = ccp_run_rsa_cmd(cmd_q, cmd);
2500	break;
2501	case CCP_ENGINE_PASSTHRU:
2502	if (cmd->flags & CCP_CMD_PASSTHRU_NO_DMA_MAP)
2503	ret = ccp_run_passthru_nomap_cmd(cmd_q, cmd);
2504	else
2505	ret = ccp_run_passthru_cmd(cmd_q, cmd);
2506	break;
2507	case CCP_ENGINE_ECC:
2508	ret = ccp_run_ecc_cmd(cmd_q, cmd);
2509	break;
2510	default:
2511	ret = -EINVAL;
2512	}
2513
2514	return ret;
2515	}
2516