1/*
2 * Cryptographic API.
3 *
4 * Glue code for the SHA256 Secure Hash Algorithm assembler
5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
6 *
7 * This file is based on sha256_generic.c
8 *
9 * Copyright (C) 2013 Intel Corporation.
10 *
11 * Author:
12 * Tim Chen <tim.c.chen@linux.intel.com>
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
17 * any later version.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * SOFTWARE.
27 */
28
29
30#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
31
32#include <crypto/internal/hash.h>
33#include <crypto/internal/simd.h>
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/mm.h>
37#include <linux/types.h>
38#include <crypto/sha2.h>
39#include <crypto/sha256_base.h>
40#include <linux/string.h>
41#include <asm/cpu_device_id.h>
42#include <asm/simd.h>
43
44asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
45 const u8 *data, int blocks);
46
47static const struct x86_cpu_id module_cpu_ids[] = {
48 X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
49 X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
50 X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
51 {}
52};
53MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
54
55static int _sha256_update(struct shash_desc *desc, const u8 *data,
56 unsigned int len, sha256_block_fn *sha256_xform)
57{
58 struct sha256_state *sctx = shash_desc_ctx(desc);
59
60 if (!crypto_simd_usable() ||
61 (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
62 return crypto_sha256_update(desc, data, len);
63
64 /*
65 * Make sure struct sha256_state begins directly with the SHA256
66 * 256-bit internal state, as this is what the asm functions expect.
67 */
68 BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
69
70 kernel_fpu_begin();
71 sha256_base_do_update(desc, data, len, block_fn: sha256_xform);
72 kernel_fpu_end();
73
74 return 0;
75}
76
77static int sha256_finup(struct shash_desc *desc, const u8 *data,
78 unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
79{
80 if (!crypto_simd_usable())
81 return crypto_sha256_finup(desc, data, len, hash: out);
82
83 kernel_fpu_begin();
84 if (len)
85 sha256_base_do_update(desc, data, len, block_fn: sha256_xform);
86 sha256_base_do_finalize(desc, block_fn: sha256_xform);
87 kernel_fpu_end();
88
89 return sha256_base_finish(desc, out);
90}
91
92static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
93 unsigned int len)
94{
95 return _sha256_update(desc, data, len, sha256_xform: sha256_transform_ssse3);
96}
97
98static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
99 unsigned int len, u8 *out)
100{
101 return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_ssse3);
102}
103
104/* Add padding and return the message digest. */
105static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
106{
107 return sha256_ssse3_finup(desc, NULL, len: 0, out);
108}
109
110static int sha256_ssse3_digest(struct shash_desc *desc, const u8 *data,
111 unsigned int len, u8 *out)
112{
113 return sha256_base_init(desc) ?:
114 sha256_ssse3_finup(desc, data, len, out);
115}
116
117static struct shash_alg sha256_ssse3_algs[] = { {
118 .digestsize = SHA256_DIGEST_SIZE,
119 .init = sha256_base_init,
120 .update = sha256_ssse3_update,
121 .final = sha256_ssse3_final,
122 .finup = sha256_ssse3_finup,
123 .digest = sha256_ssse3_digest,
124 .descsize = sizeof(struct sha256_state),
125 .base = {
126 .cra_name = "sha256",
127 .cra_driver_name = "sha256-ssse3",
128 .cra_priority = 150,
129 .cra_blocksize = SHA256_BLOCK_SIZE,
130 .cra_module = THIS_MODULE,
131 }
132}, {
133 .digestsize = SHA224_DIGEST_SIZE,
134 .init = sha224_base_init,
135 .update = sha256_ssse3_update,
136 .final = sha256_ssse3_final,
137 .finup = sha256_ssse3_finup,
138 .descsize = sizeof(struct sha256_state),
139 .base = {
140 .cra_name = "sha224",
141 .cra_driver_name = "sha224-ssse3",
142 .cra_priority = 150,
143 .cra_blocksize = SHA224_BLOCK_SIZE,
144 .cra_module = THIS_MODULE,
145 }
146} };
147
148static int register_sha256_ssse3(void)
149{
150 if (boot_cpu_has(X86_FEATURE_SSSE3))
151 return crypto_register_shashes(algs: sha256_ssse3_algs,
152 ARRAY_SIZE(sha256_ssse3_algs));
153 return 0;
154}
155
156static void unregister_sha256_ssse3(void)
157{
158 if (boot_cpu_has(X86_FEATURE_SSSE3))
159 crypto_unregister_shashes(algs: sha256_ssse3_algs,
160 ARRAY_SIZE(sha256_ssse3_algs));
161}
162
163asmlinkage void sha256_transform_avx(struct sha256_state *state,
164 const u8 *data, int blocks);
165
166static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
167 unsigned int len)
168{
169 return _sha256_update(desc, data, len, sha256_xform: sha256_transform_avx);
170}
171
172static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
173 unsigned int len, u8 *out)
174{
175 return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_avx);
176}
177
178static int sha256_avx_final(struct shash_desc *desc, u8 *out)
179{
180 return sha256_avx_finup(desc, NULL, len: 0, out);
181}
182
183static int sha256_avx_digest(struct shash_desc *desc, const u8 *data,
184 unsigned int len, u8 *out)
185{
186 return sha256_base_init(desc) ?:
187 sha256_avx_finup(desc, data, len, out);
188}
189
190static struct shash_alg sha256_avx_algs[] = { {
191 .digestsize = SHA256_DIGEST_SIZE,
192 .init = sha256_base_init,
193 .update = sha256_avx_update,
194 .final = sha256_avx_final,
195 .finup = sha256_avx_finup,
196 .digest = sha256_avx_digest,
197 .descsize = sizeof(struct sha256_state),
198 .base = {
199 .cra_name = "sha256",
200 .cra_driver_name = "sha256-avx",
201 .cra_priority = 160,
202 .cra_blocksize = SHA256_BLOCK_SIZE,
203 .cra_module = THIS_MODULE,
204 }
205}, {
206 .digestsize = SHA224_DIGEST_SIZE,
207 .init = sha224_base_init,
208 .update = sha256_avx_update,
209 .final = sha256_avx_final,
210 .finup = sha256_avx_finup,
211 .descsize = sizeof(struct sha256_state),
212 .base = {
213 .cra_name = "sha224",
214 .cra_driver_name = "sha224-avx",
215 .cra_priority = 160,
216 .cra_blocksize = SHA224_BLOCK_SIZE,
217 .cra_module = THIS_MODULE,
218 }
219} };
220
221static bool avx_usable(void)
222{
223 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
224 if (boot_cpu_has(X86_FEATURE_AVX))
225 pr_info("AVX detected but unusable.\n");
226 return false;
227 }
228
229 return true;
230}
231
232static int register_sha256_avx(void)
233{
234 if (avx_usable())
235 return crypto_register_shashes(algs: sha256_avx_algs,
236 ARRAY_SIZE(sha256_avx_algs));
237 return 0;
238}
239
240static void unregister_sha256_avx(void)
241{
242 if (avx_usable())
243 crypto_unregister_shashes(algs: sha256_avx_algs,
244 ARRAY_SIZE(sha256_avx_algs));
245}
246
247asmlinkage void sha256_transform_rorx(struct sha256_state *state,
248 const u8 *data, int blocks);
249
250static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
251 unsigned int len)
252{
253 return _sha256_update(desc, data, len, sha256_xform: sha256_transform_rorx);
254}
255
256static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
257 unsigned int len, u8 *out)
258{
259 return sha256_finup(desc, data, len, out, sha256_xform: sha256_transform_rorx);
260}
261
262static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
263{
264 return sha256_avx2_finup(desc, NULL, len: 0, out);
265}
266
267static int sha256_avx2_digest(struct shash_desc *desc, const u8 *data,
268 unsigned int len, u8 *out)
269{
270 return sha256_base_init(desc) ?:
271 sha256_avx2_finup(desc, data, len, out);
272}
273
274static struct shash_alg sha256_avx2_algs[] = { {
275 .digestsize = SHA256_DIGEST_SIZE,
276 .init = sha256_base_init,
277 .update = sha256_avx2_update,
278 .final = sha256_avx2_final,
279 .finup = sha256_avx2_finup,
280 .digest = sha256_avx2_digest,
281 .descsize = sizeof(struct sha256_state),
282 .base = {
283 .cra_name = "sha256",
284 .cra_driver_name = "sha256-avx2",
285 .cra_priority = 170,
286 .cra_blocksize = SHA256_BLOCK_SIZE,
287 .cra_module = THIS_MODULE,
288 }
289}, {
290 .digestsize = SHA224_DIGEST_SIZE,
291 .init = sha224_base_init,
292 .update = sha256_avx2_update,
293 .final = sha256_avx2_final,
294 .finup = sha256_avx2_finup,
295 .descsize = sizeof(struct sha256_state),
296 .base = {
297 .cra_name = "sha224",
298 .cra_driver_name = "sha224-avx2",
299 .cra_priority = 170,
300 .cra_blocksize = SHA224_BLOCK_SIZE,
301 .cra_module = THIS_MODULE,
302 }
303} };
304
305static bool avx2_usable(void)
306{
307 if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
308 boot_cpu_has(X86_FEATURE_BMI2))
309 return true;
310
311 return false;
312}
313
314static int register_sha256_avx2(void)
315{
316 if (avx2_usable())
317 return crypto_register_shashes(algs: sha256_avx2_algs,
318 ARRAY_SIZE(sha256_avx2_algs));
319 return 0;
320}
321
322static void unregister_sha256_avx2(void)
323{
324 if (avx2_usable())
325 crypto_unregister_shashes(algs: sha256_avx2_algs,
326 ARRAY_SIZE(sha256_avx2_algs));
327}
328
329#ifdef CONFIG_AS_SHA256_NI
330asmlinkage void sha256_ni_transform(struct sha256_state *digest,
331 const u8 *data, int rounds);
332
333static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
334 unsigned int len)
335{
336 return _sha256_update(desc, data, len, sha256_xform: sha256_ni_transform);
337}
338
339static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
340 unsigned int len, u8 *out)
341{
342 return sha256_finup(desc, data, len, out, sha256_xform: sha256_ni_transform);
343}
344
345static int sha256_ni_final(struct shash_desc *desc, u8 *out)
346{
347 return sha256_ni_finup(desc, NULL, len: 0, out);
348}
349
350static int sha256_ni_digest(struct shash_desc *desc, const u8 *data,
351 unsigned int len, u8 *out)
352{
353 return sha256_base_init(desc) ?:
354 sha256_ni_finup(desc, data, len, out);
355}
356
357static struct shash_alg sha256_ni_algs[] = { {
358 .digestsize = SHA256_DIGEST_SIZE,
359 .init = sha256_base_init,
360 .update = sha256_ni_update,
361 .final = sha256_ni_final,
362 .finup = sha256_ni_finup,
363 .digest = sha256_ni_digest,
364 .descsize = sizeof(struct sha256_state),
365 .base = {
366 .cra_name = "sha256",
367 .cra_driver_name = "sha256-ni",
368 .cra_priority = 250,
369 .cra_blocksize = SHA256_BLOCK_SIZE,
370 .cra_module = THIS_MODULE,
371 }
372}, {
373 .digestsize = SHA224_DIGEST_SIZE,
374 .init = sha224_base_init,
375 .update = sha256_ni_update,
376 .final = sha256_ni_final,
377 .finup = sha256_ni_finup,
378 .descsize = sizeof(struct sha256_state),
379 .base = {
380 .cra_name = "sha224",
381 .cra_driver_name = "sha224-ni",
382 .cra_priority = 250,
383 .cra_blocksize = SHA224_BLOCK_SIZE,
384 .cra_module = THIS_MODULE,
385 }
386} };
387
388static int register_sha256_ni(void)
389{
390 if (boot_cpu_has(X86_FEATURE_SHA_NI))
391 return crypto_register_shashes(algs: sha256_ni_algs,
392 ARRAY_SIZE(sha256_ni_algs));
393 return 0;
394}
395
396static void unregister_sha256_ni(void)
397{
398 if (boot_cpu_has(X86_FEATURE_SHA_NI))
399 crypto_unregister_shashes(algs: sha256_ni_algs,
400 ARRAY_SIZE(sha256_ni_algs));
401}
402
403#else
404static inline int register_sha256_ni(void) { return 0; }
405static inline void unregister_sha256_ni(void) { }
406#endif
407
408static int __init sha256_ssse3_mod_init(void)
409{
410 if (!x86_match_cpu(match: module_cpu_ids))
411 return -ENODEV;
412
413 if (register_sha256_ssse3())
414 goto fail;
415
416 if (register_sha256_avx()) {
417 unregister_sha256_ssse3();
418 goto fail;
419 }
420
421 if (register_sha256_avx2()) {
422 unregister_sha256_avx();
423 unregister_sha256_ssse3();
424 goto fail;
425 }
426
427 if (register_sha256_ni()) {
428 unregister_sha256_avx2();
429 unregister_sha256_avx();
430 unregister_sha256_ssse3();
431 goto fail;
432 }
433
434 return 0;
435fail:
436 return -ENODEV;
437}
438
439static void __exit sha256_ssse3_mod_fini(void)
440{
441 unregister_sha256_ni();
442 unregister_sha256_avx2();
443 unregister_sha256_avx();
444 unregister_sha256_ssse3();
445}
446
447module_init(sha256_ssse3_mod_init);
448module_exit(sha256_ssse3_mod_fini);
449
450MODULE_LICENSE("GPL");
451MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
452
453MODULE_ALIAS_CRYPTO("sha256");
454MODULE_ALIAS_CRYPTO("sha256-ssse3");
455MODULE_ALIAS_CRYPTO("sha256-avx");
456MODULE_ALIAS_CRYPTO("sha256-avx2");
457MODULE_ALIAS_CRYPTO("sha224");
458MODULE_ALIAS_CRYPTO("sha224-ssse3");
459MODULE_ALIAS_CRYPTO("sha224-avx");
460MODULE_ALIAS_CRYPTO("sha224-avx2");
461#ifdef CONFIG_AS_SHA256_NI
462MODULE_ALIAS_CRYPTO("sha256-ni");
463MODULE_ALIAS_CRYPTO("sha224-ni");
464#endif
465

source code of linux/arch/x86/crypto/sha256_ssse3_glue.c