1 | /* |
2 | * Cryptographic API. |
3 | * |
4 | * Glue code for the SHA256 Secure Hash Algorithm assembler |
5 | * implementation using supplemental SSE3 / AVX / AVX2 instructions. |
6 | * |
7 | * This file is based on sha256_generic.c |
8 | * |
9 | * Copyright (C) 2013 Intel Corporation. |
10 | * |
11 | * Author: |
12 | * Tim Chen <tim.c.chen@linux.intel.com> |
13 | * |
14 | * This program is free software; you can redistribute it and/or modify it |
15 | * under the terms of the GNU General Public License as published by the Free |
16 | * Software Foundation; either version 2 of the License, or (at your option) |
17 | * any later version. |
18 | * |
19 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
20 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
21 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
22 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS |
23 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN |
24 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
25 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
26 | * SOFTWARE. |
27 | */ |
28 | |
29 | |
30 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt |
31 | |
32 | #include <crypto/internal/hash.h> |
33 | #include <crypto/internal/simd.h> |
34 | #include <linux/init.h> |
35 | #include <linux/module.h> |
36 | #include <linux/mm.h> |
37 | #include <linux/types.h> |
38 | #include <crypto/sha2.h> |
39 | #include <crypto/sha256_base.h> |
40 | #include <linux/string.h> |
41 | #include <asm/cpu_device_id.h> |
42 | #include <asm/simd.h> |
43 | |
44 | asmlinkage void sha256_transform_ssse3(struct sha256_state *state, |
45 | const u8 *data, int blocks); |
46 | |
47 | static const struct x86_cpu_id module_cpu_ids[] = { |
48 | X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL), |
49 | X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL), |
50 | X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL), |
51 | {} |
52 | }; |
53 | MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids); |
54 | |
55 | static int _sha256_update(struct shash_desc *desc, const u8 *data, |
56 | unsigned int len, sha256_block_fn *sha256_xform) |
57 | { |
58 | struct sha256_state *sctx = shash_desc_ctx(desc); |
59 | |
60 | if (!crypto_simd_usable() || |
61 | (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE) |
62 | return crypto_sha256_update(desc, data, len); |
63 | |
64 | /* |
65 | * Make sure struct sha256_state begins directly with the SHA256 |
66 | * 256-bit internal state, as this is what the asm functions expect. |
67 | */ |
68 | BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0); |
69 | |
70 | kernel_fpu_begin(); |
71 | sha256_base_do_update(desc, data, len, block_fn: sha256_xform); |
72 | kernel_fpu_end(); |
73 | |
74 | return 0; |
75 | } |
76 | |
77 | static int sha256_finup(struct shash_desc *desc, const u8 *data, |
78 | unsigned int len, u8 *out, sha256_block_fn *sha256_xform) |
79 | { |
80 | if (!crypto_simd_usable()) |
81 | return crypto_sha256_finup(desc, data, len, hash: out); |
82 | |
83 | kernel_fpu_begin(); |
84 | if (len) |
85 | sha256_base_do_update(desc, data, len, block_fn: sha256_xform); |
86 | sha256_base_do_finalize(desc, block_fn: sha256_xform); |
87 | kernel_fpu_end(); |
88 | |
89 | return sha256_base_finish(desc, out); |
90 | } |
91 | |
92 | static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data, |
93 | unsigned int len) |
94 | { |
95 | return _sha256_update(desc, data, len, sha256_xform: sha256_transform_ssse3); |
96 | } |
97 | |
98 | static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data, |
99 | unsigned int len, u8 *out) |
100 | { |
101 | return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_ssse3); |
102 | } |
103 | |
104 | /* Add padding and return the message digest. */ |
105 | static int sha256_ssse3_final(struct shash_desc *desc, u8 *out) |
106 | { |
107 | return sha256_ssse3_finup(desc, NULL, len: 0, out); |
108 | } |
109 | |
110 | static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data, |
111 | unsigned int len, u8 *out) |
112 | { |
113 | return sha256_base_init(desc) ?: |
114 | sha256_ssse3_finup(desc, data, len, out); |
115 | } |
116 | |
117 | static struct shash_alg sha256_ssse3_algs[] = { { |
118 | .digestsize = SHA256_DIGEST_SIZE, |
119 | .init = sha256_base_init, |
120 | .update = sha256_ssse3_update, |
121 | .final = sha256_ssse3_final, |
122 | .finup = sha256_ssse3_finup, |
123 | .digest = sha256_ssse3_digest, |
124 | .descsize = sizeof(struct sha256_state), |
125 | .base = { |
126 | .cra_name = "sha256" , |
127 | .cra_driver_name = "sha256-ssse3" , |
128 | .cra_priority = 150, |
129 | .cra_blocksize = SHA256_BLOCK_SIZE, |
130 | .cra_module = THIS_MODULE, |
131 | } |
132 | }, { |
133 | .digestsize = SHA224_DIGEST_SIZE, |
134 | .init = sha224_base_init, |
135 | .update = sha256_ssse3_update, |
136 | .final = sha256_ssse3_final, |
137 | .finup = sha256_ssse3_finup, |
138 | .descsize = sizeof(struct sha256_state), |
139 | .base = { |
140 | .cra_name = "sha224" , |
141 | .cra_driver_name = "sha224-ssse3" , |
142 | .cra_priority = 150, |
143 | .cra_blocksize = SHA224_BLOCK_SIZE, |
144 | .cra_module = THIS_MODULE, |
145 | } |
146 | } }; |
147 | |
148 | static int register_sha256_ssse3(void) |
149 | { |
150 | if (boot_cpu_has(X86_FEATURE_SSSE3)) |
151 | return crypto_register_shashes(algs: sha256_ssse3_algs, |
152 | ARRAY_SIZE(sha256_ssse3_algs)); |
153 | return 0; |
154 | } |
155 | |
156 | static void unregister_sha256_ssse3(void) |
157 | { |
158 | if (boot_cpu_has(X86_FEATURE_SSSE3)) |
159 | crypto_unregister_shashes(algs: sha256_ssse3_algs, |
160 | ARRAY_SIZE(sha256_ssse3_algs)); |
161 | } |
162 | |
163 | asmlinkage void sha256_transform_avx(struct sha256_state *state, |
164 | const u8 *data, int blocks); |
165 | |
166 | static int sha256_avx_update(struct shash_desc *desc, const u8 *data, |
167 | unsigned int len) |
168 | { |
169 | return _sha256_update(desc, data, len, sha256_xform: sha256_transform_avx); |
170 | } |
171 | |
172 | static int sha256_avx_finup(struct shash_desc *desc, const u8 *data, |
173 | unsigned int len, u8 *out) |
174 | { |
175 | return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_avx); |
176 | } |
177 | |
178 | static int sha256_avx_final(struct shash_desc *desc, u8 *out) |
179 | { |
180 | return sha256_avx_finup(desc, NULL, len: 0, out); |
181 | } |
182 | |
183 | static int sha256_avx_digest(struct shash_desc *desc, const u8 *data, |
184 | unsigned int len, u8 *out) |
185 | { |
186 | return sha256_base_init(desc) ?: |
187 | sha256_avx_finup(desc, data, len, out); |
188 | } |
189 | |
190 | static struct shash_alg sha256_avx_algs[] = { { |
191 | .digestsize = SHA256_DIGEST_SIZE, |
192 | .init = sha256_base_init, |
193 | .update = sha256_avx_update, |
194 | .final = sha256_avx_final, |
195 | .finup = sha256_avx_finup, |
196 | .digest = sha256_avx_digest, |
197 | .descsize = sizeof(struct sha256_state), |
198 | .base = { |
199 | .cra_name = "sha256" , |
200 | .cra_driver_name = "sha256-avx" , |
201 | .cra_priority = 160, |
202 | .cra_blocksize = SHA256_BLOCK_SIZE, |
203 | .cra_module = THIS_MODULE, |
204 | } |
205 | }, { |
206 | .digestsize = SHA224_DIGEST_SIZE, |
207 | .init = sha224_base_init, |
208 | .update = sha256_avx_update, |
209 | .final = sha256_avx_final, |
210 | .finup = sha256_avx_finup, |
211 | .descsize = sizeof(struct sha256_state), |
212 | .base = { |
213 | .cra_name = "sha224" , |
214 | .cra_driver_name = "sha224-avx" , |
215 | .cra_priority = 160, |
216 | .cra_blocksize = SHA224_BLOCK_SIZE, |
217 | .cra_module = THIS_MODULE, |
218 | } |
219 | } }; |
220 | |
221 | static bool avx_usable(void) |
222 | { |
223 | if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) { |
224 | if (boot_cpu_has(X86_FEATURE_AVX)) |
225 | pr_info("AVX detected but unusable.\n" ); |
226 | return false; |
227 | } |
228 | |
229 | return true; |
230 | } |
231 | |
232 | static int register_sha256_avx(void) |
233 | { |
234 | if (avx_usable()) |
235 | return crypto_register_shashes(algs: sha256_avx_algs, |
236 | ARRAY_SIZE(sha256_avx_algs)); |
237 | return 0; |
238 | } |
239 | |
240 | static void unregister_sha256_avx(void) |
241 | { |
242 | if (avx_usable()) |
243 | crypto_unregister_shashes(algs: sha256_avx_algs, |
244 | ARRAY_SIZE(sha256_avx_algs)); |
245 | } |
246 | |
247 | asmlinkage void sha256_transform_rorx(struct sha256_state *state, |
248 | const u8 *data, int blocks); |
249 | |
250 | static int sha256_avx2_update(struct shash_desc *desc, const u8 *data, |
251 | unsigned int len) |
252 | { |
253 | return _sha256_update(desc, data, len, sha256_xform: sha256_transform_rorx); |
254 | } |
255 | |
256 | static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data, |
257 | unsigned int len, u8 *out) |
258 | { |
259 | return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_rorx); |
260 | } |
261 | |
262 | static int sha256_avx2_final(struct shash_desc *desc, u8 *out) |
263 | { |
264 | return sha256_avx2_finup(desc, NULL, len: 0, out); |
265 | } |
266 | |
267 | static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data, |
268 | unsigned int len, u8 *out) |
269 | { |
270 | return sha256_base_init(desc) ?: |
271 | sha256_avx2_finup(desc, data, len, out); |
272 | } |
273 | |
274 | static struct shash_alg sha256_avx2_algs[] = { { |
275 | .digestsize = SHA256_DIGEST_SIZE, |
276 | .init = sha256_base_init, |
277 | .update = sha256_avx2_update, |
278 | .final = sha256_avx2_final, |
279 | .finup = sha256_avx2_finup, |
280 | .digest = sha256_avx2_digest, |
281 | .descsize = sizeof(struct sha256_state), |
282 | .base = { |
283 | .cra_name = "sha256" , |
284 | .cra_driver_name = "sha256-avx2" , |
285 | .cra_priority = 170, |
286 | .cra_blocksize = SHA256_BLOCK_SIZE, |
287 | .cra_module = THIS_MODULE, |
288 | } |
289 | }, { |
290 | .digestsize = SHA224_DIGEST_SIZE, |
291 | .init = sha224_base_init, |
292 | .update = sha256_avx2_update, |
293 | .final = sha256_avx2_final, |
294 | .finup = sha256_avx2_finup, |
295 | .descsize = sizeof(struct sha256_state), |
296 | .base = { |
297 | .cra_name = "sha224" , |
298 | .cra_driver_name = "sha224-avx2" , |
299 | .cra_priority = 170, |
300 | .cra_blocksize = SHA224_BLOCK_SIZE, |
301 | .cra_module = THIS_MODULE, |
302 | } |
303 | } }; |
304 | |
305 | static bool avx2_usable(void) |
306 | { |
307 | if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) && |
308 | boot_cpu_has(X86_FEATURE_BMI2)) |
309 | return true; |
310 | |
311 | return false; |
312 | } |
313 | |
314 | static int register_sha256_avx2(void) |
315 | { |
316 | if (avx2_usable()) |
317 | return crypto_register_shashes(algs: sha256_avx2_algs, |
318 | ARRAY_SIZE(sha256_avx2_algs)); |
319 | return 0; |
320 | } |
321 | |
322 | static void unregister_sha256_avx2(void) |
323 | { |
324 | if (avx2_usable()) |
325 | crypto_unregister_shashes(algs: sha256_avx2_algs, |
326 | ARRAY_SIZE(sha256_avx2_algs)); |
327 | } |
328 | |
329 | #ifdef CONFIG_AS_SHA256_NI |
330 | asmlinkage void sha256_ni_transform(struct sha256_state *digest, |
331 | const u8 *data, int rounds); |
332 | |
333 | static int sha256_ni_update(struct shash_desc *desc, const u8 *data, |
334 | unsigned int len) |
335 | { |
336 | return _sha256_update(desc, data, len, sha256_xform: sha256_ni_transform); |
337 | } |
338 | |
339 | static int sha256_ni_finup(struct shash_desc *desc, const u8 *data, |
340 | unsigned int len, u8 *out) |
341 | { |
342 | return sha256_finup(desc, data, len, out, sha256_xform: sha256_ni_transform); |
343 | } |
344 | |
345 | static int sha256_ni_final(struct shash_desc *desc, u8 *out) |
346 | { |
347 | return sha256_ni_finup(desc, NULL, len: 0, out); |
348 | } |
349 | |
350 | static int sha256_ni_digest(struct shash_desc *desc, const u8 *data, |
351 | unsigned int len, u8 *out) |
352 | { |
353 | return sha256_base_init(desc) ?: |
354 | sha256_ni_finup(desc, data, len, out); |
355 | } |
356 | |
357 | static struct shash_alg sha256_ni_algs[] = { { |
358 | .digestsize = SHA256_DIGEST_SIZE, |
359 | .init = sha256_base_init, |
360 | .update = sha256_ni_update, |
361 | .final = sha256_ni_final, |
362 | .finup = sha256_ni_finup, |
363 | .digest = sha256_ni_digest, |
364 | .descsize = sizeof(struct sha256_state), |
365 | .base = { |
366 | .cra_name = "sha256" , |
367 | .cra_driver_name = "sha256-ni" , |
368 | .cra_priority = 250, |
369 | .cra_blocksize = SHA256_BLOCK_SIZE, |
370 | .cra_module = THIS_MODULE, |
371 | } |
372 | }, { |
373 | .digestsize = SHA224_DIGEST_SIZE, |
374 | .init = sha224_base_init, |
375 | .update = sha256_ni_update, |
376 | .final = sha256_ni_final, |
377 | .finup = sha256_ni_finup, |
378 | .descsize = sizeof(struct sha256_state), |
379 | .base = { |
380 | .cra_name = "sha224" , |
381 | .cra_driver_name = "sha224-ni" , |
382 | .cra_priority = 250, |
383 | .cra_blocksize = SHA224_BLOCK_SIZE, |
384 | .cra_module = THIS_MODULE, |
385 | } |
386 | } }; |
387 | |
388 | static int register_sha256_ni(void) |
389 | { |
390 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) |
391 | return crypto_register_shashes(algs: sha256_ni_algs, |
392 | ARRAY_SIZE(sha256_ni_algs)); |
393 | return 0; |
394 | } |
395 | |
396 | static void unregister_sha256_ni(void) |
397 | { |
398 | if (boot_cpu_has(X86_FEATURE_SHA_NI)) |
399 | crypto_unregister_shashes(algs: sha256_ni_algs, |
400 | ARRAY_SIZE(sha256_ni_algs)); |
401 | } |
402 | |
403 | #else |
404 | static inline int register_sha256_ni(void) { return 0; } |
405 | static inline void unregister_sha256_ni(void) { } |
406 | #endif |
407 | |
408 | static int __init sha256_ssse3_mod_init(void) |
409 | { |
410 | if (!x86_match_cpu(match: module_cpu_ids)) |
411 | return -ENODEV; |
412 | |
413 | if (register_sha256_ssse3()) |
414 | goto fail; |
415 | |
416 | if (register_sha256_avx()) { |
417 | unregister_sha256_ssse3(); |
418 | goto fail; |
419 | } |
420 | |
421 | if (register_sha256_avx2()) { |
422 | unregister_sha256_avx(); |
423 | unregister_sha256_ssse3(); |
424 | goto fail; |
425 | } |
426 | |
427 | if (register_sha256_ni()) { |
428 | unregister_sha256_avx2(); |
429 | unregister_sha256_avx(); |
430 | unregister_sha256_ssse3(); |
431 | goto fail; |
432 | } |
433 | |
434 | return 0; |
435 | fail: |
436 | return -ENODEV; |
437 | } |
438 | |
439 | static void __exit sha256_ssse3_mod_fini(void) |
440 | { |
441 | unregister_sha256_ni(); |
442 | unregister_sha256_avx2(); |
443 | unregister_sha256_avx(); |
444 | unregister_sha256_ssse3(); |
445 | } |
446 | |
447 | module_init(sha256_ssse3_mod_init); |
448 | module_exit(sha256_ssse3_mod_fini); |
449 | |
450 | MODULE_LICENSE("GPL" ); |
451 | MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated" ); |
452 | |
453 | MODULE_ALIAS_CRYPTO("sha256" ); |
454 | MODULE_ALIAS_CRYPTO("sha256-ssse3" ); |
455 | MODULE_ALIAS_CRYPTO("sha256-avx" ); |
456 | MODULE_ALIAS_CRYPTO("sha256-avx2" ); |
457 | MODULE_ALIAS_CRYPTO("sha224" ); |
458 | MODULE_ALIAS_CRYPTO("sha224-ssse3" ); |
459 | MODULE_ALIAS_CRYPTO("sha224-avx" ); |
460 | MODULE_ALIAS_CRYPTO("sha224-avx2" ); |
461 | #ifdef CONFIG_AS_SHA256_NI |
462 | MODULE_ALIAS_CRYPTO("sha256-ni" ); |
463 | MODULE_ALIAS_CRYPTO("sha224-ni" ); |
464 | #endif |
465 | |