1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * SM4 Cipher Algorithm, using ARMv8 Crypto Extensions |
4 | * as specified in |
5 | * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html |
6 | * |
7 | * Copyright (C) 2022, Alibaba Group. |
8 | * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
9 | */ |
10 | |
11 | #include <linux/module.h> |
12 | #include <linux/crypto.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/cpufeature.h> |
15 | #include <asm/neon.h> |
16 | #include <asm/simd.h> |
17 | #include <crypto/b128ops.h> |
18 | #include <crypto/internal/simd.h> |
19 | #include <crypto/internal/skcipher.h> |
20 | #include <crypto/internal/hash.h> |
21 | #include <crypto/scatterwalk.h> |
22 | #include <crypto/xts.h> |
23 | #include <crypto/sm4.h> |
24 | |
25 | #define BYTES2BLKS(nbytes) ((nbytes) >> 4) |
26 | |
27 | asmlinkage void sm4_ce_expand_key(const u8 *key, u32 *rkey_enc, u32 *rkey_dec, |
28 | const u32 *fk, const u32 *ck); |
29 | asmlinkage void sm4_ce_crypt_block(const u32 *rkey, u8 *dst, const u8 *src); |
30 | asmlinkage void sm4_ce_crypt(const u32 *rkey, u8 *dst, const u8 *src, |
31 | unsigned int nblks); |
32 | asmlinkage void sm4_ce_cbc_enc(const u32 *rkey, u8 *dst, const u8 *src, |
33 | u8 *iv, unsigned int nblocks); |
34 | asmlinkage void sm4_ce_cbc_dec(const u32 *rkey, u8 *dst, const u8 *src, |
35 | u8 *iv, unsigned int nblocks); |
36 | asmlinkage void sm4_ce_cbc_cts_enc(const u32 *rkey, u8 *dst, const u8 *src, |
37 | u8 *iv, unsigned int nbytes); |
38 | asmlinkage void sm4_ce_cbc_cts_dec(const u32 *rkey, u8 *dst, const u8 *src, |
39 | u8 *iv, unsigned int nbytes); |
40 | asmlinkage void sm4_ce_ctr_enc(const u32 *rkey, u8 *dst, const u8 *src, |
41 | u8 *iv, unsigned int nblks); |
42 | asmlinkage void sm4_ce_xts_enc(const u32 *rkey1, u8 *dst, const u8 *src, |
43 | u8 *tweak, unsigned int nbytes, |
44 | const u32 *rkey2_enc); |
45 | asmlinkage void sm4_ce_xts_dec(const u32 *rkey1, u8 *dst, const u8 *src, |
46 | u8 *tweak, unsigned int nbytes, |
47 | const u32 *rkey2_enc); |
48 | asmlinkage void sm4_ce_mac_update(const u32 *rkey_enc, u8 *digest, |
49 | const u8 *src, unsigned int nblocks, |
50 | bool enc_before, bool enc_after); |
51 | |
52 | EXPORT_SYMBOL(sm4_ce_expand_key); |
53 | EXPORT_SYMBOL(sm4_ce_crypt_block); |
54 | EXPORT_SYMBOL(sm4_ce_cbc_enc); |
55 | |
56 | struct sm4_xts_ctx { |
57 | struct sm4_ctx key1; |
58 | struct sm4_ctx key2; |
59 | }; |
60 | |
61 | struct sm4_mac_tfm_ctx { |
62 | struct sm4_ctx key; |
63 | u8 __aligned(8) consts[]; |
64 | }; |
65 | |
66 | struct sm4_mac_desc_ctx { |
67 | unsigned int len; |
68 | u8 digest[SM4_BLOCK_SIZE]; |
69 | }; |
70 | |
71 | static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, |
72 | unsigned int key_len) |
73 | { |
74 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
75 | |
76 | if (key_len != SM4_KEY_SIZE) |
77 | return -EINVAL; |
78 | |
79 | kernel_neon_begin(); |
80 | sm4_ce_expand_key(key, ctx->rkey_enc, ctx->rkey_dec, |
81 | crypto_sm4_fk, crypto_sm4_ck); |
82 | kernel_neon_end(); |
83 | return 0; |
84 | } |
85 | |
86 | static int sm4_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, |
87 | unsigned int key_len) |
88 | { |
89 | struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
90 | int ret; |
91 | |
92 | if (key_len != SM4_KEY_SIZE * 2) |
93 | return -EINVAL; |
94 | |
95 | ret = xts_verify_key(tfm, key, keylen: key_len); |
96 | if (ret) |
97 | return ret; |
98 | |
99 | kernel_neon_begin(); |
100 | sm4_ce_expand_key(key, ctx->key1.rkey_enc, |
101 | ctx->key1.rkey_dec, crypto_sm4_fk, crypto_sm4_ck); |
102 | sm4_ce_expand_key(&key[SM4_KEY_SIZE], ctx->key2.rkey_enc, |
103 | ctx->key2.rkey_dec, crypto_sm4_fk, crypto_sm4_ck); |
104 | kernel_neon_end(); |
105 | |
106 | return 0; |
107 | } |
108 | |
109 | static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey) |
110 | { |
111 | struct skcipher_walk walk; |
112 | unsigned int nbytes; |
113 | int err; |
114 | |
115 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
116 | |
117 | while ((nbytes = walk.nbytes) > 0) { |
118 | const u8 *src = walk.src.virt.addr; |
119 | u8 *dst = walk.dst.virt.addr; |
120 | unsigned int nblks; |
121 | |
122 | kernel_neon_begin(); |
123 | |
124 | nblks = BYTES2BLKS(nbytes); |
125 | if (nblks) { |
126 | sm4_ce_crypt(rkey, dst, src, nblks); |
127 | nbytes -= nblks * SM4_BLOCK_SIZE; |
128 | } |
129 | |
130 | kernel_neon_end(); |
131 | |
132 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
133 | } |
134 | |
135 | return err; |
136 | } |
137 | |
138 | static int sm4_ecb_encrypt(struct skcipher_request *req) |
139 | { |
140 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
141 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
142 | |
143 | return sm4_ecb_do_crypt(req, rkey: ctx->rkey_enc); |
144 | } |
145 | |
146 | static int sm4_ecb_decrypt(struct skcipher_request *req) |
147 | { |
148 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
149 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
150 | |
151 | return sm4_ecb_do_crypt(req, rkey: ctx->rkey_dec); |
152 | } |
153 | |
154 | static int sm4_cbc_crypt(struct skcipher_request *req, |
155 | struct sm4_ctx *ctx, bool encrypt) |
156 | { |
157 | struct skcipher_walk walk; |
158 | unsigned int nbytes; |
159 | int err; |
160 | |
161 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
162 | if (err) |
163 | return err; |
164 | |
165 | while ((nbytes = walk.nbytes) > 0) { |
166 | const u8 *src = walk.src.virt.addr; |
167 | u8 *dst = walk.dst.virt.addr; |
168 | unsigned int nblocks; |
169 | |
170 | nblocks = nbytes / SM4_BLOCK_SIZE; |
171 | if (nblocks) { |
172 | kernel_neon_begin(); |
173 | |
174 | if (encrypt) |
175 | sm4_ce_cbc_enc(ctx->rkey_enc, dst, src, |
176 | walk.iv, nblocks); |
177 | else |
178 | sm4_ce_cbc_dec(rkey: ctx->rkey_dec, dst, src, |
179 | iv: walk.iv, nblocks); |
180 | |
181 | kernel_neon_end(); |
182 | } |
183 | |
184 | err = skcipher_walk_done(walk: &walk, err: nbytes % SM4_BLOCK_SIZE); |
185 | } |
186 | |
187 | return err; |
188 | } |
189 | |
190 | static int sm4_cbc_encrypt(struct skcipher_request *req) |
191 | { |
192 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
193 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
194 | |
195 | return sm4_cbc_crypt(req, ctx, encrypt: true); |
196 | } |
197 | |
198 | static int sm4_cbc_decrypt(struct skcipher_request *req) |
199 | { |
200 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
201 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
202 | |
203 | return sm4_cbc_crypt(req, ctx, encrypt: false); |
204 | } |
205 | |
206 | static int sm4_cbc_cts_crypt(struct skcipher_request *req, bool encrypt) |
207 | { |
208 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
209 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
210 | struct scatterlist *src = req->src; |
211 | struct scatterlist *dst = req->dst; |
212 | struct scatterlist sg_src[2], sg_dst[2]; |
213 | struct skcipher_request subreq; |
214 | struct skcipher_walk walk; |
215 | int cbc_blocks; |
216 | int err; |
217 | |
218 | if (req->cryptlen < SM4_BLOCK_SIZE) |
219 | return -EINVAL; |
220 | |
221 | if (req->cryptlen == SM4_BLOCK_SIZE) |
222 | return sm4_cbc_crypt(req, ctx, encrypt); |
223 | |
224 | skcipher_request_set_tfm(req: &subreq, tfm); |
225 | skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req), |
226 | NULL, NULL); |
227 | |
228 | /* handle the CBC cryption part */ |
229 | cbc_blocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2; |
230 | if (cbc_blocks) { |
231 | skcipher_request_set_crypt(req: &subreq, src, dst, |
232 | cryptlen: cbc_blocks * SM4_BLOCK_SIZE, |
233 | iv: req->iv); |
234 | |
235 | err = sm4_cbc_crypt(req: &subreq, ctx, encrypt); |
236 | if (err) |
237 | return err; |
238 | |
239 | dst = src = scatterwalk_ffwd(dst: sg_src, src, len: subreq.cryptlen); |
240 | if (req->dst != req->src) |
241 | dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, |
242 | len: subreq.cryptlen); |
243 | } |
244 | |
245 | /* handle ciphertext stealing */ |
246 | skcipher_request_set_crypt(req: &subreq, src, dst, |
247 | cryptlen: req->cryptlen - cbc_blocks * SM4_BLOCK_SIZE, |
248 | iv: req->iv); |
249 | |
250 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
251 | if (err) |
252 | return err; |
253 | |
254 | kernel_neon_begin(); |
255 | |
256 | if (encrypt) |
257 | sm4_ce_cbc_cts_enc(rkey: ctx->rkey_enc, dst: walk.dst.virt.addr, |
258 | src: walk.src.virt.addr, iv: walk.iv, nbytes: walk.nbytes); |
259 | else |
260 | sm4_ce_cbc_cts_dec(rkey: ctx->rkey_dec, dst: walk.dst.virt.addr, |
261 | src: walk.src.virt.addr, iv: walk.iv, nbytes: walk.nbytes); |
262 | |
263 | kernel_neon_end(); |
264 | |
265 | return skcipher_walk_done(walk: &walk, err: 0); |
266 | } |
267 | |
268 | static int sm4_cbc_cts_encrypt(struct skcipher_request *req) |
269 | { |
270 | return sm4_cbc_cts_crypt(req, encrypt: true); |
271 | } |
272 | |
273 | static int sm4_cbc_cts_decrypt(struct skcipher_request *req) |
274 | { |
275 | return sm4_cbc_cts_crypt(req, encrypt: false); |
276 | } |
277 | |
278 | static int sm4_ctr_crypt(struct skcipher_request *req) |
279 | { |
280 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
281 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
282 | struct skcipher_walk walk; |
283 | unsigned int nbytes; |
284 | int err; |
285 | |
286 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
287 | |
288 | while ((nbytes = walk.nbytes) > 0) { |
289 | const u8 *src = walk.src.virt.addr; |
290 | u8 *dst = walk.dst.virt.addr; |
291 | unsigned int nblks; |
292 | |
293 | kernel_neon_begin(); |
294 | |
295 | nblks = BYTES2BLKS(nbytes); |
296 | if (nblks) { |
297 | sm4_ce_ctr_enc(rkey: ctx->rkey_enc, dst, src, iv: walk.iv, nblks); |
298 | dst += nblks * SM4_BLOCK_SIZE; |
299 | src += nblks * SM4_BLOCK_SIZE; |
300 | nbytes -= nblks * SM4_BLOCK_SIZE; |
301 | } |
302 | |
303 | /* tail */ |
304 | if (walk.nbytes == walk.total && nbytes > 0) { |
305 | u8 keystream[SM4_BLOCK_SIZE]; |
306 | |
307 | sm4_ce_crypt_block(ctx->rkey_enc, keystream, walk.iv); |
308 | crypto_inc(a: walk.iv, SM4_BLOCK_SIZE); |
309 | crypto_xor_cpy(dst, src1: src, src2: keystream, size: nbytes); |
310 | nbytes = 0; |
311 | } |
312 | |
313 | kernel_neon_end(); |
314 | |
315 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
316 | } |
317 | |
318 | return err; |
319 | } |
320 | |
321 | static int sm4_xts_crypt(struct skcipher_request *req, bool encrypt) |
322 | { |
323 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
324 | struct sm4_xts_ctx *ctx = crypto_skcipher_ctx(tfm); |
325 | int tail = req->cryptlen % SM4_BLOCK_SIZE; |
326 | const u32 *rkey2_enc = ctx->key2.rkey_enc; |
327 | struct scatterlist sg_src[2], sg_dst[2]; |
328 | struct skcipher_request subreq; |
329 | struct scatterlist *src, *dst; |
330 | struct skcipher_walk walk; |
331 | unsigned int nbytes; |
332 | int err; |
333 | |
334 | if (req->cryptlen < SM4_BLOCK_SIZE) |
335 | return -EINVAL; |
336 | |
337 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
338 | if (err) |
339 | return err; |
340 | |
341 | if (unlikely(tail > 0 && walk.nbytes < walk.total)) { |
342 | int nblocks = DIV_ROUND_UP(req->cryptlen, SM4_BLOCK_SIZE) - 2; |
343 | |
344 | skcipher_walk_abort(walk: &walk); |
345 | |
346 | skcipher_request_set_tfm(req: &subreq, tfm); |
347 | skcipher_request_set_callback(req: &subreq, |
348 | flags: skcipher_request_flags(req), |
349 | NULL, NULL); |
350 | skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst, |
351 | cryptlen: nblocks * SM4_BLOCK_SIZE, iv: req->iv); |
352 | |
353 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
354 | if (err) |
355 | return err; |
356 | } else { |
357 | tail = 0; |
358 | } |
359 | |
360 | while ((nbytes = walk.nbytes) >= SM4_BLOCK_SIZE) { |
361 | if (nbytes < walk.total) |
362 | nbytes &= ~(SM4_BLOCK_SIZE - 1); |
363 | |
364 | kernel_neon_begin(); |
365 | |
366 | if (encrypt) |
367 | sm4_ce_xts_enc(rkey1: ctx->key1.rkey_enc, dst: walk.dst.virt.addr, |
368 | src: walk.src.virt.addr, tweak: walk.iv, nbytes, |
369 | rkey2_enc); |
370 | else |
371 | sm4_ce_xts_dec(rkey1: ctx->key1.rkey_dec, dst: walk.dst.virt.addr, |
372 | src: walk.src.virt.addr, tweak: walk.iv, nbytes, |
373 | rkey2_enc); |
374 | |
375 | kernel_neon_end(); |
376 | |
377 | rkey2_enc = NULL; |
378 | |
379 | err = skcipher_walk_done(walk: &walk, err: walk.nbytes - nbytes); |
380 | if (err) |
381 | return err; |
382 | } |
383 | |
384 | if (likely(tail == 0)) |
385 | return 0; |
386 | |
387 | /* handle ciphertext stealing */ |
388 | |
389 | dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen); |
390 | if (req->dst != req->src) |
391 | dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, len: subreq.cryptlen); |
392 | |
393 | skcipher_request_set_crypt(req: &subreq, src, dst, SM4_BLOCK_SIZE + tail, |
394 | iv: req->iv); |
395 | |
396 | err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false); |
397 | if (err) |
398 | return err; |
399 | |
400 | kernel_neon_begin(); |
401 | |
402 | if (encrypt) |
403 | sm4_ce_xts_enc(rkey1: ctx->key1.rkey_enc, dst: walk.dst.virt.addr, |
404 | src: walk.src.virt.addr, tweak: walk.iv, nbytes: walk.nbytes, |
405 | rkey2_enc); |
406 | else |
407 | sm4_ce_xts_dec(rkey1: ctx->key1.rkey_dec, dst: walk.dst.virt.addr, |
408 | src: walk.src.virt.addr, tweak: walk.iv, nbytes: walk.nbytes, |
409 | rkey2_enc); |
410 | |
411 | kernel_neon_end(); |
412 | |
413 | return skcipher_walk_done(walk: &walk, err: 0); |
414 | } |
415 | |
416 | static int sm4_xts_encrypt(struct skcipher_request *req) |
417 | { |
418 | return sm4_xts_crypt(req, encrypt: true); |
419 | } |
420 | |
421 | static int sm4_xts_decrypt(struct skcipher_request *req) |
422 | { |
423 | return sm4_xts_crypt(req, encrypt: false); |
424 | } |
425 | |
426 | static struct skcipher_alg sm4_algs[] = { |
427 | { |
428 | .base = { |
429 | .cra_name = "ecb(sm4)" , |
430 | .cra_driver_name = "ecb-sm4-ce" , |
431 | .cra_priority = 400, |
432 | .cra_blocksize = SM4_BLOCK_SIZE, |
433 | .cra_ctxsize = sizeof(struct sm4_ctx), |
434 | .cra_module = THIS_MODULE, |
435 | }, |
436 | .min_keysize = SM4_KEY_SIZE, |
437 | .max_keysize = SM4_KEY_SIZE, |
438 | .setkey = sm4_setkey, |
439 | .encrypt = sm4_ecb_encrypt, |
440 | .decrypt = sm4_ecb_decrypt, |
441 | }, { |
442 | .base = { |
443 | .cra_name = "cbc(sm4)" , |
444 | .cra_driver_name = "cbc-sm4-ce" , |
445 | .cra_priority = 400, |
446 | .cra_blocksize = SM4_BLOCK_SIZE, |
447 | .cra_ctxsize = sizeof(struct sm4_ctx), |
448 | .cra_module = THIS_MODULE, |
449 | }, |
450 | .min_keysize = SM4_KEY_SIZE, |
451 | .max_keysize = SM4_KEY_SIZE, |
452 | .ivsize = SM4_BLOCK_SIZE, |
453 | .setkey = sm4_setkey, |
454 | .encrypt = sm4_cbc_encrypt, |
455 | .decrypt = sm4_cbc_decrypt, |
456 | }, { |
457 | .base = { |
458 | .cra_name = "ctr(sm4)" , |
459 | .cra_driver_name = "ctr-sm4-ce" , |
460 | .cra_priority = 400, |
461 | .cra_blocksize = 1, |
462 | .cra_ctxsize = sizeof(struct sm4_ctx), |
463 | .cra_module = THIS_MODULE, |
464 | }, |
465 | .min_keysize = SM4_KEY_SIZE, |
466 | .max_keysize = SM4_KEY_SIZE, |
467 | .ivsize = SM4_BLOCK_SIZE, |
468 | .chunksize = SM4_BLOCK_SIZE, |
469 | .setkey = sm4_setkey, |
470 | .encrypt = sm4_ctr_crypt, |
471 | .decrypt = sm4_ctr_crypt, |
472 | }, { |
473 | .base = { |
474 | .cra_name = "cts(cbc(sm4))" , |
475 | .cra_driver_name = "cts-cbc-sm4-ce" , |
476 | .cra_priority = 400, |
477 | .cra_blocksize = SM4_BLOCK_SIZE, |
478 | .cra_ctxsize = sizeof(struct sm4_ctx), |
479 | .cra_module = THIS_MODULE, |
480 | }, |
481 | .min_keysize = SM4_KEY_SIZE, |
482 | .max_keysize = SM4_KEY_SIZE, |
483 | .ivsize = SM4_BLOCK_SIZE, |
484 | .walksize = SM4_BLOCK_SIZE * 2, |
485 | .setkey = sm4_setkey, |
486 | .encrypt = sm4_cbc_cts_encrypt, |
487 | .decrypt = sm4_cbc_cts_decrypt, |
488 | }, { |
489 | .base = { |
490 | .cra_name = "xts(sm4)" , |
491 | .cra_driver_name = "xts-sm4-ce" , |
492 | .cra_priority = 400, |
493 | .cra_blocksize = SM4_BLOCK_SIZE, |
494 | .cra_ctxsize = sizeof(struct sm4_xts_ctx), |
495 | .cra_module = THIS_MODULE, |
496 | }, |
497 | .min_keysize = SM4_KEY_SIZE * 2, |
498 | .max_keysize = SM4_KEY_SIZE * 2, |
499 | .ivsize = SM4_BLOCK_SIZE, |
500 | .walksize = SM4_BLOCK_SIZE * 2, |
501 | .setkey = sm4_xts_setkey, |
502 | .encrypt = sm4_xts_encrypt, |
503 | .decrypt = sm4_xts_decrypt, |
504 | } |
505 | }; |
506 | |
507 | static int sm4_cbcmac_setkey(struct crypto_shash *tfm, const u8 *key, |
508 | unsigned int key_len) |
509 | { |
510 | struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); |
511 | |
512 | if (key_len != SM4_KEY_SIZE) |
513 | return -EINVAL; |
514 | |
515 | kernel_neon_begin(); |
516 | sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, |
517 | crypto_sm4_fk, crypto_sm4_ck); |
518 | kernel_neon_end(); |
519 | |
520 | return 0; |
521 | } |
522 | |
523 | static int sm4_cmac_setkey(struct crypto_shash *tfm, const u8 *key, |
524 | unsigned int key_len) |
525 | { |
526 | struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); |
527 | be128 *consts = (be128 *)ctx->consts; |
528 | u64 a, b; |
529 | |
530 | if (key_len != SM4_KEY_SIZE) |
531 | return -EINVAL; |
532 | |
533 | memset(consts, 0, SM4_BLOCK_SIZE); |
534 | |
535 | kernel_neon_begin(); |
536 | |
537 | sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, |
538 | crypto_sm4_fk, crypto_sm4_ck); |
539 | |
540 | /* encrypt the zero block */ |
541 | sm4_ce_crypt_block(ctx->key.rkey_enc, (u8 *)consts, (const u8 *)consts); |
542 | |
543 | kernel_neon_end(); |
544 | |
545 | /* gf(2^128) multiply zero-ciphertext with u and u^2 */ |
546 | a = be64_to_cpu(consts[0].a); |
547 | b = be64_to_cpu(consts[0].b); |
548 | consts[0].a = cpu_to_be64((a << 1) | (b >> 63)); |
549 | consts[0].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); |
550 | |
551 | a = be64_to_cpu(consts[0].a); |
552 | b = be64_to_cpu(consts[0].b); |
553 | consts[1].a = cpu_to_be64((a << 1) | (b >> 63)); |
554 | consts[1].b = cpu_to_be64((b << 1) ^ ((a >> 63) ? 0x87 : 0)); |
555 | |
556 | return 0; |
557 | } |
558 | |
559 | static int sm4_xcbc_setkey(struct crypto_shash *tfm, const u8 *key, |
560 | unsigned int key_len) |
561 | { |
562 | struct sm4_mac_tfm_ctx *ctx = crypto_shash_ctx(tfm); |
563 | u8 __aligned(8) key2[SM4_BLOCK_SIZE]; |
564 | static u8 const ks[3][SM4_BLOCK_SIZE] = { |
565 | { [0 ... SM4_BLOCK_SIZE - 1] = 0x1}, |
566 | { [0 ... SM4_BLOCK_SIZE - 1] = 0x2}, |
567 | { [0 ... SM4_BLOCK_SIZE - 1] = 0x3}, |
568 | }; |
569 | |
570 | if (key_len != SM4_KEY_SIZE) |
571 | return -EINVAL; |
572 | |
573 | kernel_neon_begin(); |
574 | |
575 | sm4_ce_expand_key(key, ctx->key.rkey_enc, ctx->key.rkey_dec, |
576 | crypto_sm4_fk, crypto_sm4_ck); |
577 | |
578 | sm4_ce_crypt_block(ctx->key.rkey_enc, key2, ks[0]); |
579 | sm4_ce_crypt(rkey: ctx->key.rkey_enc, dst: ctx->consts, src: ks[1], nblks: 2); |
580 | |
581 | sm4_ce_expand_key(key2, ctx->key.rkey_enc, ctx->key.rkey_dec, |
582 | crypto_sm4_fk, crypto_sm4_ck); |
583 | |
584 | kernel_neon_end(); |
585 | |
586 | return 0; |
587 | } |
588 | |
589 | static int sm4_mac_init(struct shash_desc *desc) |
590 | { |
591 | struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); |
592 | |
593 | memset(ctx->digest, 0, SM4_BLOCK_SIZE); |
594 | ctx->len = 0; |
595 | |
596 | return 0; |
597 | } |
598 | |
599 | static int sm4_mac_update(struct shash_desc *desc, const u8 *p, |
600 | unsigned int len) |
601 | { |
602 | struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm); |
603 | struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); |
604 | unsigned int l, nblocks; |
605 | |
606 | if (len == 0) |
607 | return 0; |
608 | |
609 | if (ctx->len || ctx->len + len < SM4_BLOCK_SIZE) { |
610 | l = min(len, SM4_BLOCK_SIZE - ctx->len); |
611 | |
612 | crypto_xor(dst: ctx->digest + ctx->len, src: p, size: l); |
613 | ctx->len += l; |
614 | len -= l; |
615 | p += l; |
616 | } |
617 | |
618 | if (len && (ctx->len % SM4_BLOCK_SIZE) == 0) { |
619 | kernel_neon_begin(); |
620 | |
621 | if (len < SM4_BLOCK_SIZE && ctx->len == SM4_BLOCK_SIZE) { |
622 | sm4_ce_crypt_block(tctx->key.rkey_enc, |
623 | ctx->digest, ctx->digest); |
624 | ctx->len = 0; |
625 | } else { |
626 | nblocks = len / SM4_BLOCK_SIZE; |
627 | len %= SM4_BLOCK_SIZE; |
628 | |
629 | sm4_ce_mac_update(rkey_enc: tctx->key.rkey_enc, digest: ctx->digest, src: p, |
630 | nblocks, enc_before: (ctx->len == SM4_BLOCK_SIZE), |
631 | enc_after: (len != 0)); |
632 | |
633 | p += nblocks * SM4_BLOCK_SIZE; |
634 | |
635 | if (len == 0) |
636 | ctx->len = SM4_BLOCK_SIZE; |
637 | } |
638 | |
639 | kernel_neon_end(); |
640 | |
641 | if (len) { |
642 | crypto_xor(dst: ctx->digest, src: p, size: len); |
643 | ctx->len = len; |
644 | } |
645 | } |
646 | |
647 | return 0; |
648 | } |
649 | |
650 | static int sm4_cmac_final(struct shash_desc *desc, u8 *out) |
651 | { |
652 | struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm); |
653 | struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); |
654 | const u8 *consts = tctx->consts; |
655 | |
656 | if (ctx->len != SM4_BLOCK_SIZE) { |
657 | ctx->digest[ctx->len] ^= 0x80; |
658 | consts += SM4_BLOCK_SIZE; |
659 | } |
660 | |
661 | kernel_neon_begin(); |
662 | sm4_ce_mac_update(rkey_enc: tctx->key.rkey_enc, digest: ctx->digest, src: consts, nblocks: 1, |
663 | enc_before: false, enc_after: true); |
664 | kernel_neon_end(); |
665 | |
666 | memcpy(out, ctx->digest, SM4_BLOCK_SIZE); |
667 | |
668 | return 0; |
669 | } |
670 | |
671 | static int sm4_cbcmac_final(struct shash_desc *desc, u8 *out) |
672 | { |
673 | struct sm4_mac_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm); |
674 | struct sm4_mac_desc_ctx *ctx = shash_desc_ctx(desc); |
675 | |
676 | if (ctx->len) { |
677 | kernel_neon_begin(); |
678 | sm4_ce_crypt_block(tctx->key.rkey_enc, ctx->digest, |
679 | ctx->digest); |
680 | kernel_neon_end(); |
681 | } |
682 | |
683 | memcpy(out, ctx->digest, SM4_BLOCK_SIZE); |
684 | |
685 | return 0; |
686 | } |
687 | |
688 | static struct shash_alg sm4_mac_algs[] = { |
689 | { |
690 | .base = { |
691 | .cra_name = "cmac(sm4)" , |
692 | .cra_driver_name = "cmac-sm4-ce" , |
693 | .cra_priority = 400, |
694 | .cra_blocksize = SM4_BLOCK_SIZE, |
695 | .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) |
696 | + SM4_BLOCK_SIZE * 2, |
697 | .cra_module = THIS_MODULE, |
698 | }, |
699 | .digestsize = SM4_BLOCK_SIZE, |
700 | .init = sm4_mac_init, |
701 | .update = sm4_mac_update, |
702 | .final = sm4_cmac_final, |
703 | .setkey = sm4_cmac_setkey, |
704 | .descsize = sizeof(struct sm4_mac_desc_ctx), |
705 | }, { |
706 | .base = { |
707 | .cra_name = "xcbc(sm4)" , |
708 | .cra_driver_name = "xcbc-sm4-ce" , |
709 | .cra_priority = 400, |
710 | .cra_blocksize = SM4_BLOCK_SIZE, |
711 | .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx) |
712 | + SM4_BLOCK_SIZE * 2, |
713 | .cra_module = THIS_MODULE, |
714 | }, |
715 | .digestsize = SM4_BLOCK_SIZE, |
716 | .init = sm4_mac_init, |
717 | .update = sm4_mac_update, |
718 | .final = sm4_cmac_final, |
719 | .setkey = sm4_xcbc_setkey, |
720 | .descsize = sizeof(struct sm4_mac_desc_ctx), |
721 | }, { |
722 | .base = { |
723 | .cra_name = "cbcmac(sm4)" , |
724 | .cra_driver_name = "cbcmac-sm4-ce" , |
725 | .cra_priority = 400, |
726 | .cra_blocksize = 1, |
727 | .cra_ctxsize = sizeof(struct sm4_mac_tfm_ctx), |
728 | .cra_module = THIS_MODULE, |
729 | }, |
730 | .digestsize = SM4_BLOCK_SIZE, |
731 | .init = sm4_mac_init, |
732 | .update = sm4_mac_update, |
733 | .final = sm4_cbcmac_final, |
734 | .setkey = sm4_cbcmac_setkey, |
735 | .descsize = sizeof(struct sm4_mac_desc_ctx), |
736 | } |
737 | }; |
738 | |
739 | static int __init sm4_init(void) |
740 | { |
741 | int err; |
742 | |
743 | err = crypto_register_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs)); |
744 | if (err) |
745 | return err; |
746 | |
747 | err = crypto_register_shashes(algs: sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs)); |
748 | if (err) |
749 | goto out_err; |
750 | |
751 | return 0; |
752 | |
753 | out_err: |
754 | crypto_unregister_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs)); |
755 | return err; |
756 | } |
757 | |
758 | static void __exit sm4_exit(void) |
759 | { |
760 | crypto_unregister_shashes(algs: sm4_mac_algs, ARRAY_SIZE(sm4_mac_algs)); |
761 | crypto_unregister_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs)); |
762 | } |
763 | |
764 | module_cpu_feature_match(SM4, sm4_init); |
765 | module_exit(sm4_exit); |
766 | |
767 | MODULE_DESCRIPTION("SM4 ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions" ); |
768 | MODULE_ALIAS_CRYPTO("sm4-ce" ); |
769 | MODULE_ALIAS_CRYPTO("sm4" ); |
770 | MODULE_ALIAS_CRYPTO("ecb(sm4)" ); |
771 | MODULE_ALIAS_CRYPTO("cbc(sm4)" ); |
772 | MODULE_ALIAS_CRYPTO("ctr(sm4)" ); |
773 | MODULE_ALIAS_CRYPTO("cts(cbc(sm4))" ); |
774 | MODULE_ALIAS_CRYPTO("xts(sm4)" ); |
775 | MODULE_ALIAS_CRYPTO("cmac(sm4)" ); |
776 | MODULE_ALIAS_CRYPTO("xcbc(sm4)" ); |
777 | MODULE_ALIAS_CRYPTO("cbcmac(sm4)" ); |
778 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>" ); |
779 | MODULE_LICENSE("GPL v2" ); |
780 | |