1 | // SPDX-License-Identifier: GPL-2.0+ |
2 | /* |
3 | * Shared descriptors for aead, skcipher algorithms |
4 | * |
5 | * Copyright 2016-2019 NXP |
6 | */ |
7 | |
8 | #include "compat.h" |
9 | #include "desc_constr.h" |
10 | #include "caamalg_desc.h" |
11 | |
12 | /* |
13 | * For aead functions, read payload and write payload, |
14 | * both of which are specified in req->src and req->dst |
15 | */ |
16 | static inline void aead_append_src_dst(u32 *desc, u32 msg_type) |
17 | { |
18 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); |
19 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH | |
20 | KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); |
21 | } |
22 | |
23 | /* Set DK bit in class 1 operation if shared */ |
24 | static inline void append_dec_op1(u32 *desc, u32 type) |
25 | { |
26 | u32 *jump_cmd, *uncond_jump_cmd; |
27 | |
28 | /* DK bit is valid only for AES */ |
29 | if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { |
30 | append_operation(desc, options: type | OP_ALG_AS_INITFINAL | |
31 | OP_ALG_DECRYPT); |
32 | return; |
33 | } |
34 | |
35 | jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); |
36 | append_operation(desc, options: type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); |
37 | uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); |
38 | set_jump_tgt_here(desc, jump_cmd); |
39 | append_operation(desc, options: type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | |
40 | OP_ALG_AAI_DK); |
41 | set_jump_tgt_here(desc, jump_cmd: uncond_jump_cmd); |
42 | } |
43 | |
44 | /** |
45 | * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor |
46 | * (non-protocol) with no (null) encryption. |
47 | * @desc: pointer to buffer used for descriptor construction |
48 | * @adata: pointer to authentication transform definitions. |
49 | * A split key is required for SEC Era < 6; the size of the split key |
50 | * is specified in this case. Valid algorithm values - one of |
51 | * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed |
52 | * with OP_ALG_AAI_HMAC_PRECOMP. |
53 | * @icvsize: integrity check value (ICV) size (truncated or full) |
54 | * @era: SEC Era |
55 | */ |
56 | void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, |
57 | unsigned int icvsize, int era) |
58 | { |
59 | u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; |
60 | |
61 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
62 | |
63 | /* Skip if already shared */ |
64 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
65 | JUMP_COND_SHRD); |
66 | if (era < 6) { |
67 | if (adata->key_inline) |
68 | append_key_as_imm(desc, data: adata->key_virt, |
69 | data_len: adata->keylen_pad, len: adata->keylen, |
70 | CLASS_2 | KEY_DEST_MDHA_SPLIT | |
71 | KEY_ENC); |
72 | else |
73 | append_key(desc, ptr: adata->key_dma, len: adata->keylen, |
74 | CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); |
75 | } else { |
76 | append_proto_dkp(desc, adata); |
77 | } |
78 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
79 | |
80 | /* assoclen + cryptlen = seqinlen */ |
81 | append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); |
82 | |
83 | /* Prepare to read and write cryptlen + assoclen bytes */ |
84 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
85 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
86 | |
87 | /* |
88 | * MOVE_LEN opcode is not available in all SEC HW revisions, |
89 | * thus need to do some magic, i.e. self-patch the descriptor |
90 | * buffer. |
91 | */ |
92 | read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | |
93 | MOVE_DEST_MATH3 | |
94 | (0x6 << MOVE_LEN_SHIFT)); |
95 | write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | |
96 | MOVE_DEST_DESCBUF | |
97 | MOVE_WAITCOMP | |
98 | (0x8 << MOVE_LEN_SHIFT)); |
99 | |
100 | /* Class 2 operation */ |
101 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
102 | OP_ALG_ENCRYPT); |
103 | |
104 | /* Read and write cryptlen bytes */ |
105 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); |
106 | |
107 | set_move_tgt_here(desc, move_cmd: read_move_cmd); |
108 | set_move_tgt_here(desc, move_cmd: write_move_cmd); |
109 | append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); |
110 | append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | |
111 | MOVE_AUX_LS); |
112 | |
113 | /* Write ICV */ |
114 | append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB | |
115 | LDST_SRCDST_BYTE_CONTEXT); |
116 | |
117 | print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": " , |
118 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
119 | 1); |
120 | } |
121 | EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); |
122 | |
123 | /** |
124 | * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor |
125 | * (non-protocol) with no (null) decryption. |
126 | * @desc: pointer to buffer used for descriptor construction |
127 | * @adata: pointer to authentication transform definitions. |
128 | * A split key is required for SEC Era < 6; the size of the split key |
129 | * is specified in this case. Valid algorithm values - one of |
130 | * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed |
131 | * with OP_ALG_AAI_HMAC_PRECOMP. |
132 | * @icvsize: integrity check value (ICV) size (truncated or full) |
133 | * @era: SEC Era |
134 | */ |
135 | void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, |
136 | unsigned int icvsize, int era) |
137 | { |
138 | u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; |
139 | |
140 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
141 | |
142 | /* Skip if already shared */ |
143 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
144 | JUMP_COND_SHRD); |
145 | if (era < 6) { |
146 | if (adata->key_inline) |
147 | append_key_as_imm(desc, data: adata->key_virt, |
148 | data_len: adata->keylen_pad, len: adata->keylen, |
149 | CLASS_2 | KEY_DEST_MDHA_SPLIT | |
150 | KEY_ENC); |
151 | else |
152 | append_key(desc, ptr: adata->key_dma, len: adata->keylen, |
153 | CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); |
154 | } else { |
155 | append_proto_dkp(desc, adata); |
156 | } |
157 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
158 | |
159 | /* Class 2 operation */ |
160 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
161 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
162 | |
163 | /* assoclen + cryptlen = seqoutlen */ |
164 | append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
165 | |
166 | /* Prepare to read and write cryptlen + assoclen bytes */ |
167 | append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); |
168 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); |
169 | |
170 | /* |
171 | * MOVE_LEN opcode is not available in all SEC HW revisions, |
172 | * thus need to do some magic, i.e. self-patch the descriptor |
173 | * buffer. |
174 | */ |
175 | read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | |
176 | MOVE_DEST_MATH2 | |
177 | (0x6 << MOVE_LEN_SHIFT)); |
178 | write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | |
179 | MOVE_DEST_DESCBUF | |
180 | MOVE_WAITCOMP | |
181 | (0x8 << MOVE_LEN_SHIFT)); |
182 | |
183 | /* Read and write cryptlen bytes */ |
184 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); |
185 | |
186 | /* |
187 | * Insert a NOP here, since we need at least 4 instructions between |
188 | * code patching the descriptor buffer and the location being patched. |
189 | */ |
190 | jump_cmd = append_jump(desc, JUMP_TEST_ALL); |
191 | set_jump_tgt_here(desc, jump_cmd); |
192 | |
193 | set_move_tgt_here(desc, move_cmd: read_move_cmd); |
194 | set_move_tgt_here(desc, move_cmd: write_move_cmd); |
195 | append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); |
196 | append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | |
197 | MOVE_AUX_LS); |
198 | append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); |
199 | |
200 | /* Load ICV */ |
201 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 | |
202 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); |
203 | |
204 | print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": " , |
205 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
206 | 1); |
207 | } |
208 | EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); |
209 | |
210 | static void init_sh_desc_key_aead(u32 * const desc, |
211 | struct alginfo * const cdata, |
212 | struct alginfo * const adata, |
213 | const bool is_rfc3686, u32 *nonce, int era) |
214 | { |
215 | u32 *key_jump_cmd; |
216 | unsigned int enckeylen = cdata->keylen; |
217 | |
218 | /* Note: Context registers are saved. */ |
219 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
220 | |
221 | /* Skip if already shared */ |
222 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
223 | JUMP_COND_SHRD); |
224 | |
225 | /* |
226 | * RFC3686 specific: |
227 | * | key = {AUTH_KEY, ENC_KEY, NONCE} |
228 | * | enckeylen = encryption key size + nonce size |
229 | */ |
230 | if (is_rfc3686) |
231 | enckeylen -= CTR_RFC3686_NONCE_SIZE; |
232 | |
233 | if (era < 6) { |
234 | if (adata->key_inline) |
235 | append_key_as_imm(desc, data: adata->key_virt, |
236 | data_len: adata->keylen_pad, len: adata->keylen, |
237 | CLASS_2 | KEY_DEST_MDHA_SPLIT | |
238 | KEY_ENC); |
239 | else |
240 | append_key(desc, ptr: adata->key_dma, len: adata->keylen, |
241 | CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); |
242 | } else { |
243 | append_proto_dkp(desc, adata); |
244 | } |
245 | |
246 | if (cdata->key_inline) |
247 | append_key_as_imm(desc, data: cdata->key_virt, data_len: enckeylen, |
248 | len: enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); |
249 | else |
250 | append_key(desc, ptr: cdata->key_dma, len: enckeylen, CLASS_1 | |
251 | KEY_DEST_CLASS_REG); |
252 | |
253 | /* Load Counter into CONTEXT1 reg */ |
254 | if (is_rfc3686) { |
255 | append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE, |
256 | LDST_CLASS_IND_CCB | |
257 | LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); |
258 | append_move(desc, |
259 | MOVE_SRC_OUTFIFO | |
260 | MOVE_DEST_CLASS1CTX | |
261 | (16 << MOVE_OFFSET_SHIFT) | |
262 | (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); |
263 | } |
264 | |
265 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
266 | } |
267 | |
268 | /** |
269 | * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor |
270 | * (non-protocol). |
271 | * @desc: pointer to buffer used for descriptor construction |
272 | * @cdata: pointer to block cipher transform definitions |
273 | * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed |
274 | * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. |
275 | * @adata: pointer to authentication transform definitions. |
276 | * A split key is required for SEC Era < 6; the size of the split key |
277 | * is specified in this case. Valid algorithm values - one of |
278 | * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed |
279 | * with OP_ALG_AAI_HMAC_PRECOMP. |
280 | * @ivsize: initialization vector size |
281 | * @icvsize: integrity check value (ICV) size (truncated or full) |
282 | * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template |
283 | * @nonce: pointer to rfc3686 nonce |
284 | * @ctx1_iv_off: IV offset in CONTEXT1 register |
285 | * @is_qi: true when called from caam/qi |
286 | * @era: SEC Era |
287 | */ |
288 | void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, |
289 | struct alginfo *adata, unsigned int ivsize, |
290 | unsigned int icvsize, const bool is_rfc3686, |
291 | u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, |
292 | int era) |
293 | { |
294 | /* Note: Context registers are saved. */ |
295 | init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); |
296 | |
297 | /* Class 2 operation */ |
298 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
299 | OP_ALG_ENCRYPT); |
300 | |
301 | if (is_qi) { |
302 | u32 *wait_load_cmd; |
303 | |
304 | /* REG3 = assoclen */ |
305 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
306 | LDST_SRCDST_WORD_DECO_MATH3 | |
307 | (4 << LDST_OFFSET_SHIFT)); |
308 | |
309 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
310 | JUMP_COND_CALM | JUMP_COND_NCP | |
311 | JUMP_COND_NOP | JUMP_COND_NIP | |
312 | JUMP_COND_NIFP); |
313 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
314 | |
315 | append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB | |
316 | LDST_SRCDST_BYTE_CONTEXT | |
317 | (ctx1_iv_off << LDST_OFFSET_SHIFT)); |
318 | } |
319 | |
320 | /* Read and write assoclen bytes */ |
321 | if (is_qi || era < 3) { |
322 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
323 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
324 | } else { |
325 | append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); |
326 | append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); |
327 | } |
328 | |
329 | /* Skip assoc data */ |
330 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
331 | |
332 | /* read assoc before reading payload */ |
333 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | |
334 | FIFOLDST_VLF); |
335 | |
336 | /* Load Counter into CONTEXT1 reg */ |
337 | if (is_rfc3686) |
338 | append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB | |
339 | LDST_SRCDST_BYTE_CONTEXT | |
340 | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << |
341 | LDST_OFFSET_SHIFT)); |
342 | |
343 | /* Class 1 operation */ |
344 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
345 | OP_ALG_ENCRYPT); |
346 | |
347 | /* Read and write cryptlen bytes */ |
348 | append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
349 | append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
350 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); |
351 | |
352 | /* Write ICV */ |
353 | append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB | |
354 | LDST_SRCDST_BYTE_CONTEXT); |
355 | |
356 | print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": " , |
357 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
358 | 1); |
359 | } |
360 | EXPORT_SYMBOL(cnstr_shdsc_aead_encap); |
361 | |
362 | /** |
363 | * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor |
364 | * (non-protocol). |
365 | * @desc: pointer to buffer used for descriptor construction |
366 | * @cdata: pointer to block cipher transform definitions |
367 | * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed |
368 | * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. |
369 | * @adata: pointer to authentication transform definitions. |
370 | * A split key is required for SEC Era < 6; the size of the split key |
371 | * is specified in this case. Valid algorithm values - one of |
372 | * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed |
373 | * with OP_ALG_AAI_HMAC_PRECOMP. |
374 | * @ivsize: initialization vector size |
375 | * @icvsize: integrity check value (ICV) size (truncated or full) |
376 | * @geniv: whether to generate Encrypted Chain IV |
377 | * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template |
378 | * @nonce: pointer to rfc3686 nonce |
379 | * @ctx1_iv_off: IV offset in CONTEXT1 register |
380 | * @is_qi: true when called from caam/qi |
381 | * @era: SEC Era |
382 | */ |
383 | void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, |
384 | struct alginfo *adata, unsigned int ivsize, |
385 | unsigned int icvsize, const bool geniv, |
386 | const bool is_rfc3686, u32 *nonce, |
387 | const u32 ctx1_iv_off, const bool is_qi, int era) |
388 | { |
389 | /* Note: Context registers are saved. */ |
390 | init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); |
391 | |
392 | /* Class 2 operation */ |
393 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
394 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
395 | |
396 | if (is_qi) { |
397 | u32 *wait_load_cmd; |
398 | |
399 | /* REG3 = assoclen */ |
400 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
401 | LDST_SRCDST_WORD_DECO_MATH3 | |
402 | (4 << LDST_OFFSET_SHIFT)); |
403 | |
404 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
405 | JUMP_COND_CALM | JUMP_COND_NCP | |
406 | JUMP_COND_NOP | JUMP_COND_NIP | |
407 | JUMP_COND_NIFP); |
408 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
409 | |
410 | if (!geniv) |
411 | append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB | |
412 | LDST_SRCDST_BYTE_CONTEXT | |
413 | (ctx1_iv_off << LDST_OFFSET_SHIFT)); |
414 | } |
415 | |
416 | /* Read and write assoclen bytes */ |
417 | if (is_qi || era < 3) { |
418 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
419 | if (geniv) |
420 | append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, |
421 | ivsize); |
422 | else |
423 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, |
424 | CAAM_CMD_SZ); |
425 | } else { |
426 | append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); |
427 | if (geniv) |
428 | append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, |
429 | ivsize); |
430 | else |
431 | append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, |
432 | CAAM_CMD_SZ); |
433 | } |
434 | |
435 | /* Skip assoc data */ |
436 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
437 | |
438 | /* read assoc before reading payload */ |
439 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | |
440 | KEY_VLF); |
441 | |
442 | if (geniv) { |
443 | append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB | |
444 | LDST_SRCDST_BYTE_CONTEXT | |
445 | (ctx1_iv_off << LDST_OFFSET_SHIFT)); |
446 | append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | |
447 | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); |
448 | } |
449 | |
450 | /* Load Counter into CONTEXT1 reg */ |
451 | if (is_rfc3686) |
452 | append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB | |
453 | LDST_SRCDST_BYTE_CONTEXT | |
454 | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << |
455 | LDST_OFFSET_SHIFT)); |
456 | |
457 | /* Choose operation */ |
458 | if (ctx1_iv_off) |
459 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
460 | OP_ALG_DECRYPT); |
461 | else |
462 | append_dec_op1(desc, type: cdata->algtype); |
463 | |
464 | /* Read and write cryptlen bytes */ |
465 | append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
466 | append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
467 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG); |
468 | |
469 | /* Load ICV */ |
470 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 | |
471 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); |
472 | |
473 | print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": " , |
474 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
475 | 1); |
476 | } |
477 | EXPORT_SYMBOL(cnstr_shdsc_aead_decap); |
478 | |
479 | /** |
480 | * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor |
481 | * (non-protocol) with HW-generated initialization |
482 | * vector. |
483 | * @desc: pointer to buffer used for descriptor construction |
484 | * @cdata: pointer to block cipher transform definitions |
485 | * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed |
486 | * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. |
487 | * @adata: pointer to authentication transform definitions. |
488 | * A split key is required for SEC Era < 6; the size of the split key |
489 | * is specified in this case. Valid algorithm values - one of |
490 | * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed |
491 | * with OP_ALG_AAI_HMAC_PRECOMP. |
492 | * @ivsize: initialization vector size |
493 | * @icvsize: integrity check value (ICV) size (truncated or full) |
494 | * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template |
495 | * @nonce: pointer to rfc3686 nonce |
496 | * @ctx1_iv_off: IV offset in CONTEXT1 register |
497 | * @is_qi: true when called from caam/qi |
498 | * @era: SEC Era |
499 | */ |
500 | void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, |
501 | struct alginfo *adata, unsigned int ivsize, |
502 | unsigned int icvsize, const bool is_rfc3686, |
503 | u32 *nonce, const u32 ctx1_iv_off, |
504 | const bool is_qi, int era) |
505 | { |
506 | u32 geniv, moveiv; |
507 | u32 *wait_cmd; |
508 | |
509 | /* Note: Context registers are saved. */ |
510 | init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); |
511 | |
512 | if (is_qi) { |
513 | u32 *wait_load_cmd; |
514 | |
515 | /* REG3 = assoclen */ |
516 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
517 | LDST_SRCDST_WORD_DECO_MATH3 | |
518 | (4 << LDST_OFFSET_SHIFT)); |
519 | |
520 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
521 | JUMP_COND_CALM | JUMP_COND_NCP | |
522 | JUMP_COND_NOP | JUMP_COND_NIP | |
523 | JUMP_COND_NIFP); |
524 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
525 | } |
526 | |
527 | if (is_rfc3686) { |
528 | if (is_qi) |
529 | append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB | |
530 | LDST_SRCDST_BYTE_CONTEXT | |
531 | (ctx1_iv_off << LDST_OFFSET_SHIFT)); |
532 | |
533 | goto copy_iv; |
534 | } |
535 | |
536 | /* Generate IV */ |
537 | geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | |
538 | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | |
539 | NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); |
540 | append_load_imm_u32(desc, immediate: geniv, LDST_CLASS_IND_CCB | |
541 | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); |
542 | append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); |
543 | append_move(desc, MOVE_WAITCOMP | |
544 | MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | |
545 | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | |
546 | (ivsize << MOVE_LEN_SHIFT)); |
547 | append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); |
548 | |
549 | copy_iv: |
550 | /* Copy IV to class 1 context */ |
551 | append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | |
552 | (ctx1_iv_off << MOVE_OFFSET_SHIFT) | |
553 | (ivsize << MOVE_LEN_SHIFT)); |
554 | |
555 | /* Return to encryption */ |
556 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
557 | OP_ALG_ENCRYPT); |
558 | |
559 | /* Read and write assoclen bytes */ |
560 | if (is_qi || era < 3) { |
561 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
562 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
563 | } else { |
564 | append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); |
565 | append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); |
566 | } |
567 | |
568 | /* Skip assoc data */ |
569 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
570 | |
571 | /* read assoc before reading payload */ |
572 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | |
573 | KEY_VLF); |
574 | |
575 | /* Copy iv from outfifo to class 2 fifo */ |
576 | moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | |
577 | NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); |
578 | append_load_imm_u32(desc, immediate: moveiv, LDST_CLASS_IND_CCB | |
579 | LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); |
580 | append_load_imm_u32(desc, immediate: ivsize, LDST_CLASS_2_CCB | |
581 | LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); |
582 | |
583 | /* Load Counter into CONTEXT1 reg */ |
584 | if (is_rfc3686) |
585 | append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB | |
586 | LDST_SRCDST_BYTE_CONTEXT | |
587 | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << |
588 | LDST_OFFSET_SHIFT)); |
589 | |
590 | /* Class 1 operation */ |
591 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
592 | OP_ALG_ENCRYPT); |
593 | |
594 | /* Will write ivsize + cryptlen */ |
595 | append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
596 | |
597 | /* Not need to reload iv */ |
598 | append_seq_fifo_load(desc, len: ivsize, |
599 | FIFOLD_CLASS_SKIP); |
600 | |
601 | /* Will read cryptlen */ |
602 | append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
603 | |
604 | /* |
605 | * Wait for IV transfer (ofifo -> class2) to finish before starting |
606 | * ciphertext transfer (ofifo -> external memory). |
607 | */ |
608 | wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NIFP); |
609 | set_jump_tgt_here(desc, jump_cmd: wait_cmd); |
610 | |
611 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH | KEY_VLF | |
612 | FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); |
613 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); |
614 | |
615 | /* Write ICV */ |
616 | append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB | |
617 | LDST_SRCDST_BYTE_CONTEXT); |
618 | |
619 | print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": " , |
620 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
621 | 1); |
622 | } |
623 | EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); |
624 | |
625 | /** |
626 | * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor |
627 | * @desc: pointer to buffer used for descriptor construction |
628 | * @cdata: pointer to block cipher transform definitions |
629 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
630 | * @ivsize: initialization vector size |
631 | * @icvsize: integrity check value (ICV) size (truncated or full) |
632 | * @is_qi: true when called from caam/qi |
633 | */ |
634 | void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, |
635 | unsigned int ivsize, unsigned int icvsize, |
636 | const bool is_qi) |
637 | { |
638 | u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, |
639 | *zero_assoc_jump_cmd2; |
640 | |
641 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
642 | |
643 | /* skip key loading if they are loaded due to sharing */ |
644 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
645 | JUMP_COND_SHRD); |
646 | if (cdata->key_inline) |
647 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
648 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
649 | else |
650 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
651 | KEY_DEST_CLASS_REG); |
652 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
653 | |
654 | /* class 1 operation */ |
655 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
656 | OP_ALG_ENCRYPT); |
657 | |
658 | if (is_qi) { |
659 | u32 *wait_load_cmd; |
660 | |
661 | /* REG3 = assoclen */ |
662 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
663 | LDST_SRCDST_WORD_DECO_MATH3 | |
664 | (4 << LDST_OFFSET_SHIFT)); |
665 | |
666 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
667 | JUMP_COND_CALM | JUMP_COND_NCP | |
668 | JUMP_COND_NOP | JUMP_COND_NIP | |
669 | JUMP_COND_NIFP); |
670 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
671 | |
672 | append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, |
673 | ivsize); |
674 | } else { |
675 | append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, |
676 | CAAM_CMD_SZ); |
677 | } |
678 | |
679 | /* if assoclen + cryptlen is ZERO, skip to ICV write */ |
680 | zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | |
681 | JUMP_COND_MATH_Z); |
682 | |
683 | if (is_qi) |
684 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
685 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
686 | |
687 | /* if assoclen is ZERO, skip reading the assoc data */ |
688 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
689 | zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | |
690 | JUMP_COND_MATH_Z); |
691 | |
692 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
693 | |
694 | /* skip assoc data */ |
695 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
696 | |
697 | /* cryptlen = seqinlen - assoclen */ |
698 | append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); |
699 | |
700 | /* if cryptlen is ZERO jump to zero-payload commands */ |
701 | zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | |
702 | JUMP_COND_MATH_Z); |
703 | |
704 | /* read assoc data */ |
705 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
706 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); |
707 | set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd1); |
708 | |
709 | append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
710 | |
711 | /* write encrypted data */ |
712 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); |
713 | |
714 | /* read payload data */ |
715 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
716 | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); |
717 | |
718 | /* jump to ICV writing */ |
719 | if (is_qi) |
720 | append_jump(desc, JUMP_TEST_ALL | 4); |
721 | else |
722 | append_jump(desc, JUMP_TEST_ALL | 2); |
723 | |
724 | /* zero-payload commands */ |
725 | set_jump_tgt_here(desc, jump_cmd: zero_payload_jump_cmd); |
726 | |
727 | /* read assoc data */ |
728 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
729 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); |
730 | if (is_qi) |
731 | /* jump to ICV writing */ |
732 | append_jump(desc, JUMP_TEST_ALL | 2); |
733 | |
734 | /* There is no input data */ |
735 | set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd2); |
736 | |
737 | if (is_qi) |
738 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
739 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | |
740 | FIFOLD_TYPE_LAST1); |
741 | |
742 | /* write ICV */ |
743 | append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB | |
744 | LDST_SRCDST_BYTE_CONTEXT); |
745 | |
746 | print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": " , |
747 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
748 | 1); |
749 | } |
750 | EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); |
751 | |
752 | /** |
753 | * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor |
754 | * @desc: pointer to buffer used for descriptor construction |
755 | * @cdata: pointer to block cipher transform definitions |
756 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
757 | * @ivsize: initialization vector size |
758 | * @icvsize: integrity check value (ICV) size (truncated or full) |
759 | * @is_qi: true when called from caam/qi |
760 | */ |
761 | void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, |
762 | unsigned int ivsize, unsigned int icvsize, |
763 | const bool is_qi) |
764 | { |
765 | u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; |
766 | |
767 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
768 | |
769 | /* skip key loading if they are loaded due to sharing */ |
770 | key_jump_cmd = append_jump(desc, JUMP_JSL | |
771 | JUMP_TEST_ALL | JUMP_COND_SHRD); |
772 | if (cdata->key_inline) |
773 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
774 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
775 | else |
776 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
777 | KEY_DEST_CLASS_REG); |
778 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
779 | |
780 | /* class 1 operation */ |
781 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
782 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
783 | |
784 | if (is_qi) { |
785 | u32 *wait_load_cmd; |
786 | |
787 | /* REG3 = assoclen */ |
788 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
789 | LDST_SRCDST_WORD_DECO_MATH3 | |
790 | (4 << LDST_OFFSET_SHIFT)); |
791 | |
792 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
793 | JUMP_COND_CALM | JUMP_COND_NCP | |
794 | JUMP_COND_NOP | JUMP_COND_NIP | |
795 | JUMP_COND_NIFP); |
796 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
797 | |
798 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
799 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
800 | } |
801 | |
802 | /* if assoclen is ZERO, skip reading the assoc data */ |
803 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
804 | zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | |
805 | JUMP_COND_MATH_Z); |
806 | |
807 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
808 | |
809 | /* skip assoc data */ |
810 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
811 | |
812 | /* read assoc data */ |
813 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
814 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); |
815 | |
816 | set_jump_tgt_here(desc, jump_cmd: zero_assoc_jump_cmd1); |
817 | |
818 | /* cryptlen = seqoutlen - assoclen */ |
819 | append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
820 | |
821 | /* jump to zero-payload command if cryptlen is zero */ |
822 | zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | |
823 | JUMP_COND_MATH_Z); |
824 | |
825 | append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
826 | |
827 | /* store encrypted data */ |
828 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); |
829 | |
830 | /* read payload data */ |
831 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
832 | FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); |
833 | |
834 | /* zero-payload command */ |
835 | set_jump_tgt_here(desc, jump_cmd: zero_payload_jump_cmd); |
836 | |
837 | /* read ICV */ |
838 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 | |
839 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); |
840 | |
841 | print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": " , |
842 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
843 | 1); |
844 | } |
845 | EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); |
846 | |
847 | /** |
848 | * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor |
849 | * (non-protocol). |
850 | * @desc: pointer to buffer used for descriptor construction |
851 | * @cdata: pointer to block cipher transform definitions |
852 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
853 | * @ivsize: initialization vector size |
854 | * @icvsize: integrity check value (ICV) size (truncated or full) |
855 | * @is_qi: true when called from caam/qi |
856 | * |
857 | * Input sequence: AAD | PTXT |
858 | * Output sequence: AAD | CTXT | ICV |
859 | * AAD length (assoclen), which includes the IV length, is available in Math3. |
860 | */ |
861 | void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, |
862 | unsigned int ivsize, unsigned int icvsize, |
863 | const bool is_qi) |
864 | { |
865 | u32 *key_jump_cmd, *zero_cryptlen_jump_cmd, *skip_instructions; |
866 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
867 | |
868 | /* Skip key loading if it is loaded due to sharing */ |
869 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
870 | JUMP_COND_SHRD); |
871 | if (cdata->key_inline) |
872 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
873 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
874 | else |
875 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
876 | KEY_DEST_CLASS_REG); |
877 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
878 | |
879 | /* Class 1 operation */ |
880 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
881 | OP_ALG_ENCRYPT); |
882 | |
883 | if (is_qi) { |
884 | u32 *wait_load_cmd; |
885 | |
886 | /* REG3 = assoclen */ |
887 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
888 | LDST_SRCDST_WORD_DECO_MATH3 | |
889 | (4 << LDST_OFFSET_SHIFT)); |
890 | |
891 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
892 | JUMP_COND_CALM | JUMP_COND_NCP | |
893 | JUMP_COND_NOP | JUMP_COND_NIP | |
894 | JUMP_COND_NIFP); |
895 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
896 | |
897 | /* Read salt and IV */ |
898 | append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt + |
899 | cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 | |
900 | FIFOLD_TYPE_IV); |
901 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
902 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
903 | } |
904 | |
905 | append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); |
906 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
907 | |
908 | /* Skip AAD */ |
909 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
910 | |
911 | /* Read cryptlen and set this value into VARSEQOUTLEN */ |
912 | append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); |
913 | |
914 | /* If cryptlen is ZERO jump to AAD command */ |
915 | zero_cryptlen_jump_cmd = append_jump(desc, JUMP_TEST_ALL | |
916 | JUMP_COND_MATH_Z); |
917 | |
918 | /* Read AAD data */ |
919 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
920 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); |
921 | |
922 | /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ |
923 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA); |
924 | |
925 | /* Skip IV */ |
926 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_SKIP); |
927 | append_math_add(desc, VARSEQINLEN, VARSEQOUTLEN, REG0, CAAM_CMD_SZ); |
928 | |
929 | /* Write encrypted data */ |
930 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); |
931 | |
932 | /* Read payload data */ |
933 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
934 | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); |
935 | |
936 | /* Jump instructions to avoid double reading of AAD */ |
937 | skip_instructions = append_jump(desc, JUMP_TEST_ALL); |
938 | |
939 | /* There is no input data, cryptlen = 0 */ |
940 | set_jump_tgt_here(desc, jump_cmd: zero_cryptlen_jump_cmd); |
941 | |
942 | /* Read AAD */ |
943 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
944 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); |
945 | |
946 | set_jump_tgt_here(desc, jump_cmd: skip_instructions); |
947 | |
948 | /* Write ICV */ |
949 | append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB | |
950 | LDST_SRCDST_BYTE_CONTEXT); |
951 | |
952 | print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": " , |
953 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
954 | 1); |
955 | } |
956 | EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); |
957 | |
958 | /** |
959 | * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor |
960 | * (non-protocol). |
961 | * @desc: pointer to buffer used for descriptor construction |
962 | * @cdata: pointer to block cipher transform definitions |
963 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
964 | * @ivsize: initialization vector size |
965 | * @icvsize: integrity check value (ICV) size (truncated or full) |
966 | * @is_qi: true when called from caam/qi |
967 | */ |
968 | void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, |
969 | unsigned int ivsize, unsigned int icvsize, |
970 | const bool is_qi) |
971 | { |
972 | u32 *key_jump_cmd; |
973 | |
974 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
975 | |
976 | /* Skip key loading if it is loaded due to sharing */ |
977 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
978 | JUMP_COND_SHRD); |
979 | if (cdata->key_inline) |
980 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
981 | len: cdata->keylen, CLASS_1 | |
982 | KEY_DEST_CLASS_REG); |
983 | else |
984 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
985 | KEY_DEST_CLASS_REG); |
986 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
987 | |
988 | /* Class 1 operation */ |
989 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
990 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
991 | |
992 | if (is_qi) { |
993 | u32 *wait_load_cmd; |
994 | |
995 | /* REG3 = assoclen */ |
996 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
997 | LDST_SRCDST_WORD_DECO_MATH3 | |
998 | (4 << LDST_OFFSET_SHIFT)); |
999 | |
1000 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1001 | JUMP_COND_CALM | JUMP_COND_NCP | |
1002 | JUMP_COND_NOP | JUMP_COND_NIP | |
1003 | JUMP_COND_NIFP); |
1004 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
1005 | |
1006 | /* Read salt and IV */ |
1007 | append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt + |
1008 | cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 | |
1009 | FIFOLD_TYPE_IV); |
1010 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
1011 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
1012 | } |
1013 | |
1014 | append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); |
1015 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
1016 | |
1017 | /* Read assoc data */ |
1018 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
1019 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); |
1020 | |
1021 | /* Skip IV */ |
1022 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_SKIP); |
1023 | |
1024 | /* Will read cryptlen bytes */ |
1025 | append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); |
1026 | |
1027 | /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ |
1028 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); |
1029 | |
1030 | /* Skip assoc data */ |
1031 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); |
1032 | |
1033 | /* Will write cryptlen bytes */ |
1034 | append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
1035 | |
1036 | /* Store payload data */ |
1037 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); |
1038 | |
1039 | /* Read encrypted data */ |
1040 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | |
1041 | FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); |
1042 | |
1043 | /* Read ICV */ |
1044 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 | |
1045 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); |
1046 | |
1047 | print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": " , |
1048 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1049 | 1); |
1050 | } |
1051 | EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); |
1052 | |
1053 | /** |
1054 | * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor |
1055 | * (non-protocol). |
1056 | * @desc: pointer to buffer used for descriptor construction |
1057 | * @cdata: pointer to block cipher transform definitions |
1058 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
1059 | * @ivsize: initialization vector size |
1060 | * @icvsize: integrity check value (ICV) size (truncated or full) |
1061 | * @is_qi: true when called from caam/qi |
1062 | */ |
1063 | void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, |
1064 | unsigned int ivsize, unsigned int icvsize, |
1065 | const bool is_qi) |
1066 | { |
1067 | u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; |
1068 | |
1069 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
1070 | |
1071 | /* Skip key loading if it is loaded due to sharing */ |
1072 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1073 | JUMP_COND_SHRD); |
1074 | if (cdata->key_inline) |
1075 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1076 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1077 | else |
1078 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
1079 | KEY_DEST_CLASS_REG); |
1080 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1081 | |
1082 | /* Class 1 operation */ |
1083 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
1084 | OP_ALG_ENCRYPT); |
1085 | |
1086 | if (is_qi) { |
1087 | /* assoclen is not needed, skip it */ |
1088 | append_seq_fifo_load(desc, len: 4, FIFOLD_CLASS_SKIP); |
1089 | |
1090 | /* Read salt and IV */ |
1091 | append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt + |
1092 | cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 | |
1093 | FIFOLD_TYPE_IV); |
1094 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
1095 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
1096 | } |
1097 | |
1098 | /* assoclen + cryptlen = seqinlen */ |
1099 | append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); |
1100 | |
1101 | /* |
1102 | * MOVE_LEN opcode is not available in all SEC HW revisions, |
1103 | * thus need to do some magic, i.e. self-patch the descriptor |
1104 | * buffer. |
1105 | */ |
1106 | read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | |
1107 | (0x6 << MOVE_LEN_SHIFT)); |
1108 | write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | |
1109 | (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); |
1110 | |
1111 | /* Will read assoclen + cryptlen bytes */ |
1112 | append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
1113 | |
1114 | /* Will write assoclen + cryptlen bytes */ |
1115 | append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
1116 | |
1117 | /* Read and write assoclen + cryptlen bytes */ |
1118 | aead_append_src_dst(desc, FIFOLD_TYPE_AAD); |
1119 | |
1120 | set_move_tgt_here(desc, move_cmd: read_move_cmd); |
1121 | set_move_tgt_here(desc, move_cmd: write_move_cmd); |
1122 | append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); |
1123 | /* Move payload data to OFIFO */ |
1124 | append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); |
1125 | |
1126 | /* Write ICV */ |
1127 | append_seq_store(desc, len: icvsize, LDST_CLASS_1_CCB | |
1128 | LDST_SRCDST_BYTE_CONTEXT); |
1129 | |
1130 | print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": " , |
1131 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1132 | 1); |
1133 | } |
1134 | EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); |
1135 | |
1136 | /** |
1137 | * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor |
1138 | * (non-protocol). |
1139 | * @desc: pointer to buffer used for descriptor construction |
1140 | * @cdata: pointer to block cipher transform definitions |
1141 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. |
1142 | * @ivsize: initialization vector size |
1143 | * @icvsize: integrity check value (ICV) size (truncated or full) |
1144 | * @is_qi: true when called from caam/qi |
1145 | */ |
1146 | void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, |
1147 | unsigned int ivsize, unsigned int icvsize, |
1148 | const bool is_qi) |
1149 | { |
1150 | u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; |
1151 | |
1152 | init_sh_desc(desc, HDR_SHARE_SERIAL); |
1153 | |
1154 | /* Skip key loading if it is loaded due to sharing */ |
1155 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1156 | JUMP_COND_SHRD); |
1157 | if (cdata->key_inline) |
1158 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1159 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1160 | else |
1161 | append_key(desc, ptr: cdata->key_dma, len: cdata->keylen, CLASS_1 | |
1162 | KEY_DEST_CLASS_REG); |
1163 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1164 | |
1165 | /* Class 1 operation */ |
1166 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
1167 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
1168 | |
1169 | if (is_qi) { |
1170 | /* assoclen is not needed, skip it */ |
1171 | append_seq_fifo_load(desc, len: 4, FIFOLD_CLASS_SKIP); |
1172 | |
1173 | /* Read salt and IV */ |
1174 | append_fifo_load_as_imm(desc, data: (void *)(cdata->key_virt + |
1175 | cdata->keylen), len: 4, FIFOLD_CLASS_CLASS1 | |
1176 | FIFOLD_TYPE_IV); |
1177 | append_seq_fifo_load(desc, len: ivsize, FIFOLD_CLASS_CLASS1 | |
1178 | FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); |
1179 | } |
1180 | |
1181 | /* assoclen + cryptlen = seqoutlen */ |
1182 | append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
1183 | |
1184 | /* |
1185 | * MOVE_LEN opcode is not available in all SEC HW revisions, |
1186 | * thus need to do some magic, i.e. self-patch the descriptor |
1187 | * buffer. |
1188 | */ |
1189 | read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | |
1190 | (0x6 << MOVE_LEN_SHIFT)); |
1191 | write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | |
1192 | (0x8 << MOVE_LEN_SHIFT) | MOVE_WAITCOMP); |
1193 | |
1194 | /* Will read assoclen + cryptlen bytes */ |
1195 | append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
1196 | |
1197 | /* Will write assoclen + cryptlen bytes */ |
1198 | append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); |
1199 | |
1200 | /* Store payload data */ |
1201 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); |
1202 | |
1203 | /* In-snoop assoclen + cryptlen data */ |
1204 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | |
1205 | FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); |
1206 | |
1207 | set_move_tgt_here(desc, move_cmd: read_move_cmd); |
1208 | set_move_tgt_here(desc, move_cmd: write_move_cmd); |
1209 | append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); |
1210 | /* Move payload data to OFIFO */ |
1211 | append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); |
1212 | append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); |
1213 | |
1214 | /* Read ICV */ |
1215 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS1 | |
1216 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); |
1217 | |
1218 | print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": " , |
1219 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1220 | 1); |
1221 | } |
1222 | EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); |
1223 | |
1224 | /** |
1225 | * cnstr_shdsc_chachapoly - Chacha20 + Poly1305 generic AEAD (rfc7539) and |
1226 | * IPsec ESP (rfc7634, a.k.a. rfc7539esp) shared |
1227 | * descriptor (non-protocol). |
1228 | * @desc: pointer to buffer used for descriptor construction |
1229 | * @cdata: pointer to block cipher transform definitions |
1230 | * Valid algorithm values - OP_ALG_ALGSEL_CHACHA20 ANDed with |
1231 | * OP_ALG_AAI_AEAD. |
1232 | * @adata: pointer to authentication transform definitions |
1233 | * Valid algorithm values - OP_ALG_ALGSEL_POLY1305 ANDed with |
1234 | * OP_ALG_AAI_AEAD. |
1235 | * @ivsize: initialization vector size |
1236 | * @icvsize: integrity check value (ICV) size (truncated or full) |
1237 | * @encap: true if encapsulation, false if decapsulation |
1238 | * @is_qi: true when called from caam/qi |
1239 | */ |
1240 | void cnstr_shdsc_chachapoly(u32 * const desc, struct alginfo *cdata, |
1241 | struct alginfo *adata, unsigned int ivsize, |
1242 | unsigned int icvsize, const bool encap, |
1243 | const bool is_qi) |
1244 | { |
1245 | u32 *key_jump_cmd, *wait_cmd; |
1246 | u32 nfifo; |
1247 | const bool is_ipsec = (ivsize != CHACHAPOLY_IV_SIZE); |
1248 | |
1249 | /* Note: Context registers are saved. */ |
1250 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
1251 | |
1252 | /* skip key loading if they are loaded due to sharing */ |
1253 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1254 | JUMP_COND_SHRD); |
1255 | |
1256 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, len: cdata->keylen, |
1257 | CLASS_1 | KEY_DEST_CLASS_REG); |
1258 | |
1259 | /* For IPsec load the salt from keymat in the context register */ |
1260 | if (is_ipsec) |
1261 | append_load_as_imm(desc, data: cdata->key_virt + cdata->keylen, len: 4, |
1262 | LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT | |
1263 | 4 << LDST_OFFSET_SHIFT); |
1264 | |
1265 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1266 | |
1267 | /* Class 2 and 1 operations: Poly & ChaCha */ |
1268 | if (encap) { |
1269 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
1270 | OP_ALG_ENCRYPT); |
1271 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
1272 | OP_ALG_ENCRYPT); |
1273 | } else { |
1274 | append_operation(desc, options: adata->algtype | OP_ALG_AS_INITFINAL | |
1275 | OP_ALG_DECRYPT | OP_ALG_ICV_ON); |
1276 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
1277 | OP_ALG_DECRYPT); |
1278 | } |
1279 | |
1280 | if (is_qi) { |
1281 | u32 *wait_load_cmd; |
1282 | u32 ctx1_iv_off = is_ipsec ? 8 : 4; |
1283 | |
1284 | /* REG3 = assoclen */ |
1285 | append_seq_load(desc, len: 4, LDST_CLASS_DECO | |
1286 | LDST_SRCDST_WORD_DECO_MATH3 | |
1287 | 4 << LDST_OFFSET_SHIFT); |
1288 | |
1289 | wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1290 | JUMP_COND_CALM | JUMP_COND_NCP | |
1291 | JUMP_COND_NOP | JUMP_COND_NIP | |
1292 | JUMP_COND_NIFP); |
1293 | set_jump_tgt_here(desc, jump_cmd: wait_load_cmd); |
1294 | |
1295 | append_seq_load(desc, len: ivsize, LDST_CLASS_1_CCB | |
1296 | LDST_SRCDST_BYTE_CONTEXT | |
1297 | ctx1_iv_off << LDST_OFFSET_SHIFT); |
1298 | } |
1299 | |
1300 | /* |
1301 | * MAGIC with NFIFO |
1302 | * Read associated data from the input and send them to class1 and |
1303 | * class2 alignment blocks. From class1 send data to output fifo and |
1304 | * then write it to memory since we don't need to encrypt AD. |
1305 | */ |
1306 | nfifo = NFIFOENTRY_DEST_BOTH | NFIFOENTRY_FC1 | NFIFOENTRY_FC2 | |
1307 | NFIFOENTRY_DTYPE_POLY | NFIFOENTRY_BND; |
1308 | append_load_imm_u32(desc, immediate: nfifo, LDST_CLASS_IND_CCB | |
1309 | LDST_SRCDST_WORD_INFO_FIFO_SM | LDLEN_MATH3); |
1310 | |
1311 | append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); |
1312 | append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); |
1313 | append_seq_fifo_load(desc, len: 0, FIFOLD_TYPE_NOINFOFIFO | |
1314 | FIFOLD_CLASS_CLASS1 | LDST_VLF); |
1315 | append_move_len(desc, MOVE_AUX_LS | MOVE_SRC_AUX_ABLK | |
1316 | MOVE_DEST_OUTFIFO | MOVELEN_MRSEL_MATH3); |
1317 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); |
1318 | |
1319 | /* IPsec - copy IV at the output */ |
1320 | if (is_ipsec) |
1321 | append_seq_fifo_store(desc, len: ivsize, FIFOST_TYPE_METADATA | |
1322 | 0x2 << 25); |
1323 | |
1324 | wait_cmd = append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | |
1325 | JUMP_COND_NOP | JUMP_TEST_ALL); |
1326 | set_jump_tgt_here(desc, jump_cmd: wait_cmd); |
1327 | |
1328 | if (encap) { |
1329 | /* Read and write cryptlen bytes */ |
1330 | append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
1331 | append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, |
1332 | CAAM_CMD_SZ); |
1333 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); |
1334 | |
1335 | /* Write ICV */ |
1336 | append_seq_store(desc, len: icvsize, LDST_CLASS_2_CCB | |
1337 | LDST_SRCDST_BYTE_CONTEXT); |
1338 | } else { |
1339 | /* Read and write cryptlen bytes */ |
1340 | append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, |
1341 | CAAM_CMD_SZ); |
1342 | append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, |
1343 | CAAM_CMD_SZ); |
1344 | aead_append_src_dst(desc, FIFOLD_TYPE_MSG); |
1345 | |
1346 | /* Load ICV for verification */ |
1347 | append_seq_fifo_load(desc, len: icvsize, FIFOLD_CLASS_CLASS2 | |
1348 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); |
1349 | } |
1350 | |
1351 | print_hex_dump_debug("chachapoly shdesc@" __stringify(__LINE__)": " , |
1352 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1353 | 1); |
1354 | } |
1355 | EXPORT_SYMBOL(cnstr_shdsc_chachapoly); |
1356 | |
1357 | /* For skcipher encrypt and decrypt, read from req->src and write to req->dst */ |
1358 | static inline void skcipher_append_src_dst(u32 *desc) |
1359 | { |
1360 | append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
1361 | append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); |
1362 | append_seq_fifo_load(desc, len: 0, FIFOLD_CLASS_CLASS1 | |
1363 | KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); |
1364 | append_seq_fifo_store(desc, len: 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); |
1365 | } |
1366 | |
1367 | /** |
1368 | * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor |
1369 | * @desc: pointer to buffer used for descriptor construction |
1370 | * @cdata: pointer to block cipher transform definitions |
1371 | * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed |
1372 | * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 |
1373 | * - OP_ALG_ALGSEL_CHACHA20 |
1374 | * @ivsize: initialization vector size |
1375 | * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template |
1376 | * @ctx1_iv_off: IV offset in CONTEXT1 register |
1377 | */ |
1378 | void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, |
1379 | unsigned int ivsize, const bool is_rfc3686, |
1380 | const u32 ctx1_iv_off) |
1381 | { |
1382 | u32 *key_jump_cmd; |
1383 | u32 options = cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT; |
1384 | bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == |
1385 | OP_ALG_ALGSEL_CHACHA20); |
1386 | |
1387 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
1388 | /* Skip if already shared */ |
1389 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1390 | JUMP_COND_SHRD); |
1391 | |
1392 | /* Load class1 key only */ |
1393 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1394 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1395 | |
1396 | /* Load nonce into CONTEXT1 reg */ |
1397 | if (is_rfc3686) { |
1398 | const u8 *nonce = cdata->key_virt + cdata->keylen; |
1399 | |
1400 | append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE, |
1401 | LDST_CLASS_IND_CCB | |
1402 | LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); |
1403 | append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | |
1404 | MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | |
1405 | (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); |
1406 | } |
1407 | |
1408 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1409 | |
1410 | /* Load IV, if there is one */ |
1411 | if (ivsize) |
1412 | append_seq_load(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT | |
1413 | LDST_CLASS_1_CCB | (ctx1_iv_off << |
1414 | LDST_OFFSET_SHIFT)); |
1415 | |
1416 | /* Load counter into CONTEXT1 reg */ |
1417 | if (is_rfc3686) |
1418 | append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB | |
1419 | LDST_SRCDST_BYTE_CONTEXT | |
1420 | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << |
1421 | LDST_OFFSET_SHIFT)); |
1422 | |
1423 | /* Load operation */ |
1424 | if (is_chacha20) |
1425 | options |= OP_ALG_AS_FINALIZE; |
1426 | append_operation(desc, options); |
1427 | |
1428 | /* Perform operation */ |
1429 | skcipher_append_src_dst(desc); |
1430 | |
1431 | /* Store IV */ |
1432 | if (!is_chacha20 && ivsize) |
1433 | append_seq_store(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT | |
1434 | LDST_CLASS_1_CCB | (ctx1_iv_off << |
1435 | LDST_OFFSET_SHIFT)); |
1436 | |
1437 | print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": " , |
1438 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1439 | 1); |
1440 | } |
1441 | EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); |
1442 | |
1443 | /** |
1444 | * cnstr_shdsc_skcipher_decap - skcipher decapsulation shared descriptor |
1445 | * @desc: pointer to buffer used for descriptor construction |
1446 | * @cdata: pointer to block cipher transform definitions |
1447 | * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed |
1448 | * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128 |
1449 | * - OP_ALG_ALGSEL_CHACHA20 |
1450 | * @ivsize: initialization vector size |
1451 | * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template |
1452 | * @ctx1_iv_off: IV offset in CONTEXT1 register |
1453 | */ |
1454 | void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, |
1455 | unsigned int ivsize, const bool is_rfc3686, |
1456 | const u32 ctx1_iv_off) |
1457 | { |
1458 | u32 *key_jump_cmd; |
1459 | bool is_chacha20 = ((cdata->algtype & OP_ALG_ALGSEL_MASK) == |
1460 | OP_ALG_ALGSEL_CHACHA20); |
1461 | |
1462 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
1463 | /* Skip if already shared */ |
1464 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1465 | JUMP_COND_SHRD); |
1466 | |
1467 | /* Load class1 key only */ |
1468 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1469 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1470 | |
1471 | /* Load nonce into CONTEXT1 reg */ |
1472 | if (is_rfc3686) { |
1473 | const u8 *nonce = cdata->key_virt + cdata->keylen; |
1474 | |
1475 | append_load_as_imm(desc, data: nonce, CTR_RFC3686_NONCE_SIZE, |
1476 | LDST_CLASS_IND_CCB | |
1477 | LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); |
1478 | append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | |
1479 | MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | |
1480 | (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); |
1481 | } |
1482 | |
1483 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1484 | |
1485 | /* Load IV, if there is one */ |
1486 | if (ivsize) |
1487 | append_seq_load(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT | |
1488 | LDST_CLASS_1_CCB | (ctx1_iv_off << |
1489 | LDST_OFFSET_SHIFT)); |
1490 | |
1491 | /* Load counter into CONTEXT1 reg */ |
1492 | if (is_rfc3686) |
1493 | append_load_imm_be32(desc, immediate: 1, LDST_IMM | LDST_CLASS_1_CCB | |
1494 | LDST_SRCDST_BYTE_CONTEXT | |
1495 | ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << |
1496 | LDST_OFFSET_SHIFT)); |
1497 | |
1498 | /* Choose operation */ |
1499 | if (ctx1_iv_off) |
1500 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INIT | |
1501 | OP_ALG_DECRYPT); |
1502 | else |
1503 | append_dec_op1(desc, type: cdata->algtype); |
1504 | |
1505 | /* Perform operation */ |
1506 | skcipher_append_src_dst(desc); |
1507 | |
1508 | /* Store IV */ |
1509 | if (!is_chacha20 && ivsize) |
1510 | append_seq_store(desc, len: ivsize, LDST_SRCDST_BYTE_CONTEXT | |
1511 | LDST_CLASS_1_CCB | (ctx1_iv_off << |
1512 | LDST_OFFSET_SHIFT)); |
1513 | |
1514 | print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": " , |
1515 | DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), |
1516 | 1); |
1517 | } |
1518 | EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); |
1519 | |
1520 | /** |
1521 | * cnstr_shdsc_xts_skcipher_encap - xts skcipher encapsulation shared descriptor |
1522 | * @desc: pointer to buffer used for descriptor construction |
1523 | * @cdata: pointer to block cipher transform definitions |
1524 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. |
1525 | */ |
1526 | void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) |
1527 | { |
1528 | /* |
1529 | * Set sector size to a big value, practically disabling |
1530 | * sector size segmentation in xts implementation. We cannot |
1531 | * take full advantage of this HW feature with existing |
1532 | * crypto API / dm-crypt SW architecture. |
1533 | */ |
1534 | __be64 sector_size = cpu_to_be64(BIT(15)); |
1535 | u32 *key_jump_cmd; |
1536 | |
1537 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
1538 | /* Skip if already shared */ |
1539 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1540 | JUMP_COND_SHRD); |
1541 | |
1542 | /* Load class1 keys only */ |
1543 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1544 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1545 | |
1546 | /* Load sector size with index 40 bytes (0x28) */ |
1547 | append_load_as_imm(desc, data: (void *)§or_size, len: 8, LDST_CLASS_1_CCB | |
1548 | LDST_SRCDST_BYTE_CONTEXT | |
1549 | (0x28 << LDST_OFFSET_SHIFT)); |
1550 | |
1551 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1552 | |
1553 | /* |
1554 | * create sequence for loading the sector index / 16B tweak value |
1555 | * Lower 8B of IV - sector index / tweak lower half |
1556 | * Upper 8B of IV - upper half of 16B tweak |
1557 | */ |
1558 | append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1559 | (0x20 << LDST_OFFSET_SHIFT)); |
1560 | append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1561 | (0x30 << LDST_OFFSET_SHIFT)); |
1562 | |
1563 | /* Load operation */ |
1564 | append_operation(desc, options: cdata->algtype | OP_ALG_AS_INITFINAL | |
1565 | OP_ALG_ENCRYPT); |
1566 | |
1567 | /* Perform operation */ |
1568 | skcipher_append_src_dst(desc); |
1569 | |
1570 | /* Store lower 8B and upper 8B of IV */ |
1571 | append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1572 | (0x20 << LDST_OFFSET_SHIFT)); |
1573 | append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1574 | (0x30 << LDST_OFFSET_SHIFT)); |
1575 | |
1576 | print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) |
1577 | ": " , DUMP_PREFIX_ADDRESS, 16, 4, |
1578 | desc, desc_bytes(desc), 1); |
1579 | } |
1580 | EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); |
1581 | |
1582 | /** |
1583 | * cnstr_shdsc_xts_skcipher_decap - xts skcipher decapsulation shared descriptor |
1584 | * @desc: pointer to buffer used for descriptor construction |
1585 | * @cdata: pointer to block cipher transform definitions |
1586 | * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. |
1587 | */ |
1588 | void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) |
1589 | { |
1590 | /* |
1591 | * Set sector size to a big value, practically disabling |
1592 | * sector size segmentation in xts implementation. We cannot |
1593 | * take full advantage of this HW feature with existing |
1594 | * crypto API / dm-crypt SW architecture. |
1595 | */ |
1596 | __be64 sector_size = cpu_to_be64(BIT(15)); |
1597 | u32 *key_jump_cmd; |
1598 | |
1599 | init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); |
1600 | /* Skip if already shared */ |
1601 | key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | |
1602 | JUMP_COND_SHRD); |
1603 | |
1604 | /* Load class1 key only */ |
1605 | append_key_as_imm(desc, data: cdata->key_virt, data_len: cdata->keylen, |
1606 | len: cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); |
1607 | |
1608 | /* Load sector size with index 40 bytes (0x28) */ |
1609 | append_load_as_imm(desc, data: (void *)§or_size, len: 8, LDST_CLASS_1_CCB | |
1610 | LDST_SRCDST_BYTE_CONTEXT | |
1611 | (0x28 << LDST_OFFSET_SHIFT)); |
1612 | |
1613 | set_jump_tgt_here(desc, jump_cmd: key_jump_cmd); |
1614 | |
1615 | /* |
1616 | * create sequence for loading the sector index / 16B tweak value |
1617 | * Lower 8B of IV - sector index / tweak lower half |
1618 | * Upper 8B of IV - upper half of 16B tweak |
1619 | */ |
1620 | append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1621 | (0x20 << LDST_OFFSET_SHIFT)); |
1622 | append_seq_load(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1623 | (0x30 << LDST_OFFSET_SHIFT)); |
1624 | /* Load operation */ |
1625 | append_dec_op1(desc, type: cdata->algtype); |
1626 | |
1627 | /* Perform operation */ |
1628 | skcipher_append_src_dst(desc); |
1629 | |
1630 | /* Store lower 8B and upper 8B of IV */ |
1631 | append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1632 | (0x20 << LDST_OFFSET_SHIFT)); |
1633 | append_seq_store(desc, len: 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | |
1634 | (0x30 << LDST_OFFSET_SHIFT)); |
1635 | |
1636 | print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) |
1637 | ": " , DUMP_PREFIX_ADDRESS, 16, 4, desc, |
1638 | desc_bytes(desc), 1); |
1639 | } |
1640 | EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); |
1641 | |
1642 | MODULE_LICENSE("GPL" ); |
1643 | MODULE_DESCRIPTION("FSL CAAM descriptor support" ); |
1644 | MODULE_AUTHOR("Freescale Semiconductor - NMG/STC" ); |
1645 | |