1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * SM4 Cipher Algorithm, AES-NI/AVX2 optimized. |
4 | * as specified in |
5 | * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html |
6 | * |
7 | * Copyright (c) 2021, Alibaba Group. |
8 | * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
9 | */ |
10 | |
11 | #include <linux/module.h> |
12 | #include <linux/crypto.h> |
13 | #include <linux/kernel.h> |
14 | #include <asm/simd.h> |
15 | #include <crypto/internal/simd.h> |
16 | #include <crypto/internal/skcipher.h> |
17 | #include <crypto/sm4.h> |
18 | #include "sm4-avx.h" |
19 | |
20 | #define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16) |
21 | |
22 | asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst, |
23 | const u8 *src, u8 *iv); |
24 | asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst, |
25 | const u8 *src, u8 *iv); |
26 | asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst, |
27 | const u8 *src, u8 *iv); |
28 | |
29 | static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, |
30 | unsigned int key_len) |
31 | { |
32 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
33 | |
34 | return sm4_expandkey(ctx, in_key: key, key_len); |
35 | } |
36 | |
37 | static int cbc_decrypt(struct skcipher_request *req) |
38 | { |
39 | return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, |
40 | func: sm4_aesni_avx2_cbc_dec_blk16); |
41 | } |
42 | |
43 | |
44 | static int cfb_decrypt(struct skcipher_request *req) |
45 | { |
46 | return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE, |
47 | func: sm4_aesni_avx2_cfb_dec_blk16); |
48 | } |
49 | |
50 | static int ctr_crypt(struct skcipher_request *req) |
51 | { |
52 | return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE, |
53 | func: sm4_aesni_avx2_ctr_enc_blk16); |
54 | } |
55 | |
56 | static struct skcipher_alg sm4_aesni_avx2_skciphers[] = { |
57 | { |
58 | .base = { |
59 | .cra_name = "__ecb(sm4)" , |
60 | .cra_driver_name = "__ecb-sm4-aesni-avx2" , |
61 | .cra_priority = 500, |
62 | .cra_flags = CRYPTO_ALG_INTERNAL, |
63 | .cra_blocksize = SM4_BLOCK_SIZE, |
64 | .cra_ctxsize = sizeof(struct sm4_ctx), |
65 | .cra_module = THIS_MODULE, |
66 | }, |
67 | .min_keysize = SM4_KEY_SIZE, |
68 | .max_keysize = SM4_KEY_SIZE, |
69 | .walksize = 16 * SM4_BLOCK_SIZE, |
70 | .setkey = sm4_skcipher_setkey, |
71 | .encrypt = sm4_avx_ecb_encrypt, |
72 | .decrypt = sm4_avx_ecb_decrypt, |
73 | }, { |
74 | .base = { |
75 | .cra_name = "__cbc(sm4)" , |
76 | .cra_driver_name = "__cbc-sm4-aesni-avx2" , |
77 | .cra_priority = 500, |
78 | .cra_flags = CRYPTO_ALG_INTERNAL, |
79 | .cra_blocksize = SM4_BLOCK_SIZE, |
80 | .cra_ctxsize = sizeof(struct sm4_ctx), |
81 | .cra_module = THIS_MODULE, |
82 | }, |
83 | .min_keysize = SM4_KEY_SIZE, |
84 | .max_keysize = SM4_KEY_SIZE, |
85 | .ivsize = SM4_BLOCK_SIZE, |
86 | .walksize = 16 * SM4_BLOCK_SIZE, |
87 | .setkey = sm4_skcipher_setkey, |
88 | .encrypt = sm4_cbc_encrypt, |
89 | .decrypt = cbc_decrypt, |
90 | }, { |
91 | .base = { |
92 | .cra_name = "__cfb(sm4)" , |
93 | .cra_driver_name = "__cfb-sm4-aesni-avx2" , |
94 | .cra_priority = 500, |
95 | .cra_flags = CRYPTO_ALG_INTERNAL, |
96 | .cra_blocksize = 1, |
97 | .cra_ctxsize = sizeof(struct sm4_ctx), |
98 | .cra_module = THIS_MODULE, |
99 | }, |
100 | .min_keysize = SM4_KEY_SIZE, |
101 | .max_keysize = SM4_KEY_SIZE, |
102 | .ivsize = SM4_BLOCK_SIZE, |
103 | .chunksize = SM4_BLOCK_SIZE, |
104 | .walksize = 16 * SM4_BLOCK_SIZE, |
105 | .setkey = sm4_skcipher_setkey, |
106 | .encrypt = sm4_cfb_encrypt, |
107 | .decrypt = cfb_decrypt, |
108 | }, { |
109 | .base = { |
110 | .cra_name = "__ctr(sm4)" , |
111 | .cra_driver_name = "__ctr-sm4-aesni-avx2" , |
112 | .cra_priority = 500, |
113 | .cra_flags = CRYPTO_ALG_INTERNAL, |
114 | .cra_blocksize = 1, |
115 | .cra_ctxsize = sizeof(struct sm4_ctx), |
116 | .cra_module = THIS_MODULE, |
117 | }, |
118 | .min_keysize = SM4_KEY_SIZE, |
119 | .max_keysize = SM4_KEY_SIZE, |
120 | .ivsize = SM4_BLOCK_SIZE, |
121 | .chunksize = SM4_BLOCK_SIZE, |
122 | .walksize = 16 * SM4_BLOCK_SIZE, |
123 | .setkey = sm4_skcipher_setkey, |
124 | .encrypt = ctr_crypt, |
125 | .decrypt = ctr_crypt, |
126 | } |
127 | }; |
128 | |
129 | static struct simd_skcipher_alg * |
130 | simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)]; |
131 | |
132 | static int __init sm4_init(void) |
133 | { |
134 | const char *feature_name; |
135 | |
136 | if (!boot_cpu_has(X86_FEATURE_AVX) || |
137 | !boot_cpu_has(X86_FEATURE_AVX2) || |
138 | !boot_cpu_has(X86_FEATURE_AES) || |
139 | !boot_cpu_has(X86_FEATURE_OSXSAVE)) { |
140 | pr_info("AVX2 or AES-NI instructions are not detected.\n" ); |
141 | return -ENODEV; |
142 | } |
143 | |
144 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, |
145 | feature_name: &feature_name)) { |
146 | pr_info("CPU feature '%s' is not supported.\n" , feature_name); |
147 | return -ENODEV; |
148 | } |
149 | |
150 | return simd_register_skciphers_compat(algs: sm4_aesni_avx2_skciphers, |
151 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), |
152 | simd_algs: simd_sm4_aesni_avx2_skciphers); |
153 | } |
154 | |
155 | static void __exit sm4_exit(void) |
156 | { |
157 | simd_unregister_skciphers(algs: sm4_aesni_avx2_skciphers, |
158 | ARRAY_SIZE(sm4_aesni_avx2_skciphers), |
159 | simd_algs: simd_sm4_aesni_avx2_skciphers); |
160 | } |
161 | |
162 | module_init(sm4_init); |
163 | module_exit(sm4_exit); |
164 | |
165 | MODULE_LICENSE("GPL v2" ); |
166 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>" ); |
167 | MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized" ); |
168 | MODULE_ALIAS_CRYPTO("sm4" ); |
169 | MODULE_ALIAS_CRYPTO("sm4-aesni-avx2" ); |
170 | |