1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
2 | /* |
3 | * SM4 Cipher Algorithm, using ARMv8 NEON |
4 | * as specified in |
5 | * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html |
6 | * |
7 | * Copyright (C) 2022, Alibaba Group. |
8 | * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com> |
9 | */ |
10 | |
11 | #include <linux/module.h> |
12 | #include <linux/crypto.h> |
13 | #include <linux/kernel.h> |
14 | #include <linux/cpufeature.h> |
15 | #include <asm/neon.h> |
16 | #include <asm/simd.h> |
17 | #include <crypto/internal/simd.h> |
18 | #include <crypto/internal/skcipher.h> |
19 | #include <crypto/sm4.h> |
20 | |
21 | asmlinkage void sm4_neon_crypt(const u32 *rkey, u8 *dst, const u8 *src, |
22 | unsigned int nblocks); |
23 | asmlinkage void sm4_neon_cbc_dec(const u32 *rkey_dec, u8 *dst, const u8 *src, |
24 | u8 *iv, unsigned int nblocks); |
25 | asmlinkage void sm4_neon_ctr_crypt(const u32 *rkey_enc, u8 *dst, const u8 *src, |
26 | u8 *iv, unsigned int nblocks); |
27 | |
28 | static int sm4_setkey(struct crypto_skcipher *tfm, const u8 *key, |
29 | unsigned int key_len) |
30 | { |
31 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
32 | |
33 | return sm4_expandkey(ctx, in_key: key, key_len); |
34 | } |
35 | |
36 | static int sm4_ecb_do_crypt(struct skcipher_request *req, const u32 *rkey) |
37 | { |
38 | struct skcipher_walk walk; |
39 | unsigned int nbytes; |
40 | int err; |
41 | |
42 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
43 | |
44 | while ((nbytes = walk.nbytes) > 0) { |
45 | const u8 *src = walk.src.virt.addr; |
46 | u8 *dst = walk.dst.virt.addr; |
47 | unsigned int nblocks; |
48 | |
49 | nblocks = nbytes / SM4_BLOCK_SIZE; |
50 | if (nblocks) { |
51 | kernel_neon_begin(); |
52 | |
53 | sm4_neon_crypt(rkey, dst, src, nblocks); |
54 | |
55 | kernel_neon_end(); |
56 | } |
57 | |
58 | err = skcipher_walk_done(walk: &walk, err: nbytes % SM4_BLOCK_SIZE); |
59 | } |
60 | |
61 | return err; |
62 | } |
63 | |
64 | static int sm4_ecb_encrypt(struct skcipher_request *req) |
65 | { |
66 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
67 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
68 | |
69 | return sm4_ecb_do_crypt(req, rkey: ctx->rkey_enc); |
70 | } |
71 | |
72 | static int sm4_ecb_decrypt(struct skcipher_request *req) |
73 | { |
74 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
75 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
76 | |
77 | return sm4_ecb_do_crypt(req, rkey: ctx->rkey_dec); |
78 | } |
79 | |
80 | static int sm4_cbc_encrypt(struct skcipher_request *req) |
81 | { |
82 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
83 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
84 | struct skcipher_walk walk; |
85 | unsigned int nbytes; |
86 | int err; |
87 | |
88 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
89 | |
90 | while ((nbytes = walk.nbytes) > 0) { |
91 | const u8 *iv = walk.iv; |
92 | const u8 *src = walk.src.virt.addr; |
93 | u8 *dst = walk.dst.virt.addr; |
94 | |
95 | while (nbytes >= SM4_BLOCK_SIZE) { |
96 | crypto_xor_cpy(dst, src1: src, src2: iv, SM4_BLOCK_SIZE); |
97 | sm4_crypt_block(rk: ctx->rkey_enc, out: dst, in: dst); |
98 | iv = dst; |
99 | src += SM4_BLOCK_SIZE; |
100 | dst += SM4_BLOCK_SIZE; |
101 | nbytes -= SM4_BLOCK_SIZE; |
102 | } |
103 | if (iv != walk.iv) |
104 | memcpy(walk.iv, iv, SM4_BLOCK_SIZE); |
105 | |
106 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
107 | } |
108 | |
109 | return err; |
110 | } |
111 | |
112 | static int sm4_cbc_decrypt(struct skcipher_request *req) |
113 | { |
114 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
115 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
116 | struct skcipher_walk walk; |
117 | unsigned int nbytes; |
118 | int err; |
119 | |
120 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
121 | |
122 | while ((nbytes = walk.nbytes) > 0) { |
123 | const u8 *src = walk.src.virt.addr; |
124 | u8 *dst = walk.dst.virt.addr; |
125 | unsigned int nblocks; |
126 | |
127 | nblocks = nbytes / SM4_BLOCK_SIZE; |
128 | if (nblocks) { |
129 | kernel_neon_begin(); |
130 | |
131 | sm4_neon_cbc_dec(rkey_dec: ctx->rkey_dec, dst, src, |
132 | iv: walk.iv, nblocks); |
133 | |
134 | kernel_neon_end(); |
135 | } |
136 | |
137 | err = skcipher_walk_done(walk: &walk, err: nbytes % SM4_BLOCK_SIZE); |
138 | } |
139 | |
140 | return err; |
141 | } |
142 | |
143 | static int sm4_ctr_crypt(struct skcipher_request *req) |
144 | { |
145 | struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); |
146 | struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm); |
147 | struct skcipher_walk walk; |
148 | unsigned int nbytes; |
149 | int err; |
150 | |
151 | err = skcipher_walk_virt(walk: &walk, req, atomic: false); |
152 | |
153 | while ((nbytes = walk.nbytes) > 0) { |
154 | const u8 *src = walk.src.virt.addr; |
155 | u8 *dst = walk.dst.virt.addr; |
156 | unsigned int nblocks; |
157 | |
158 | nblocks = nbytes / SM4_BLOCK_SIZE; |
159 | if (nblocks) { |
160 | kernel_neon_begin(); |
161 | |
162 | sm4_neon_ctr_crypt(rkey_enc: ctx->rkey_enc, dst, src, |
163 | iv: walk.iv, nblocks); |
164 | |
165 | kernel_neon_end(); |
166 | |
167 | dst += nblocks * SM4_BLOCK_SIZE; |
168 | src += nblocks * SM4_BLOCK_SIZE; |
169 | nbytes -= nblocks * SM4_BLOCK_SIZE; |
170 | } |
171 | |
172 | /* tail */ |
173 | if (walk.nbytes == walk.total && nbytes > 0) { |
174 | u8 keystream[SM4_BLOCK_SIZE]; |
175 | |
176 | sm4_crypt_block(rk: ctx->rkey_enc, out: keystream, in: walk.iv); |
177 | crypto_inc(a: walk.iv, SM4_BLOCK_SIZE); |
178 | crypto_xor_cpy(dst, src1: src, src2: keystream, size: nbytes); |
179 | nbytes = 0; |
180 | } |
181 | |
182 | err = skcipher_walk_done(walk: &walk, err: nbytes); |
183 | } |
184 | |
185 | return err; |
186 | } |
187 | |
188 | static struct skcipher_alg sm4_algs[] = { |
189 | { |
190 | .base = { |
191 | .cra_name = "ecb(sm4)" , |
192 | .cra_driver_name = "ecb-sm4-neon" , |
193 | .cra_priority = 200, |
194 | .cra_blocksize = SM4_BLOCK_SIZE, |
195 | .cra_ctxsize = sizeof(struct sm4_ctx), |
196 | .cra_module = THIS_MODULE, |
197 | }, |
198 | .min_keysize = SM4_KEY_SIZE, |
199 | .max_keysize = SM4_KEY_SIZE, |
200 | .setkey = sm4_setkey, |
201 | .encrypt = sm4_ecb_encrypt, |
202 | .decrypt = sm4_ecb_decrypt, |
203 | }, { |
204 | .base = { |
205 | .cra_name = "cbc(sm4)" , |
206 | .cra_driver_name = "cbc-sm4-neon" , |
207 | .cra_priority = 200, |
208 | .cra_blocksize = SM4_BLOCK_SIZE, |
209 | .cra_ctxsize = sizeof(struct sm4_ctx), |
210 | .cra_module = THIS_MODULE, |
211 | }, |
212 | .min_keysize = SM4_KEY_SIZE, |
213 | .max_keysize = SM4_KEY_SIZE, |
214 | .ivsize = SM4_BLOCK_SIZE, |
215 | .setkey = sm4_setkey, |
216 | .encrypt = sm4_cbc_encrypt, |
217 | .decrypt = sm4_cbc_decrypt, |
218 | }, { |
219 | .base = { |
220 | .cra_name = "ctr(sm4)" , |
221 | .cra_driver_name = "ctr-sm4-neon" , |
222 | .cra_priority = 200, |
223 | .cra_blocksize = 1, |
224 | .cra_ctxsize = sizeof(struct sm4_ctx), |
225 | .cra_module = THIS_MODULE, |
226 | }, |
227 | .min_keysize = SM4_KEY_SIZE, |
228 | .max_keysize = SM4_KEY_SIZE, |
229 | .ivsize = SM4_BLOCK_SIZE, |
230 | .chunksize = SM4_BLOCK_SIZE, |
231 | .setkey = sm4_setkey, |
232 | .encrypt = sm4_ctr_crypt, |
233 | .decrypt = sm4_ctr_crypt, |
234 | } |
235 | }; |
236 | |
237 | static int __init sm4_init(void) |
238 | { |
239 | return crypto_register_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs)); |
240 | } |
241 | |
242 | static void __exit sm4_exit(void) |
243 | { |
244 | crypto_unregister_skciphers(algs: sm4_algs, ARRAY_SIZE(sm4_algs)); |
245 | } |
246 | |
247 | module_init(sm4_init); |
248 | module_exit(sm4_exit); |
249 | |
250 | MODULE_DESCRIPTION("SM4 ECB/CBC/CTR using ARMv8 NEON" ); |
251 | MODULE_ALIAS_CRYPTO("sm4-neon" ); |
252 | MODULE_ALIAS_CRYPTO("sm4" ); |
253 | MODULE_ALIAS_CRYPTO("ecb(sm4)" ); |
254 | MODULE_ALIAS_CRYPTO("cbc(sm4)" ); |
255 | MODULE_ALIAS_CRYPTO("ctr(sm4)" ); |
256 | MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>" ); |
257 | MODULE_LICENSE("GPL v2" ); |
258 | |