1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
4 | */ |
5 | |
6 | #include <crypto/internal/blake2s.h> |
7 | |
8 | #include <linux/types.h> |
9 | #include <linux/jump_label.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/sizes.h> |
12 | |
13 | #include <asm/cpufeature.h> |
14 | #include <asm/fpu/api.h> |
15 | #include <asm/processor.h> |
16 | #include <asm/simd.h> |
17 | |
18 | asmlinkage void blake2s_compress_ssse3(struct blake2s_state *state, |
19 | const u8 *block, const size_t nblocks, |
20 | const u32 inc); |
21 | asmlinkage void blake2s_compress_avx512(struct blake2s_state *state, |
22 | const u8 *block, const size_t nblocks, |
23 | const u32 inc); |
24 | |
25 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_ssse3); |
26 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(blake2s_use_avx512); |
27 | |
28 | void blake2s_compress(struct blake2s_state *state, const u8 *block, |
29 | size_t nblocks, const u32 inc) |
30 | { |
31 | /* SIMD disables preemption, so relax after processing each page. */ |
32 | BUILD_BUG_ON(SZ_4K / BLAKE2S_BLOCK_SIZE < 8); |
33 | |
34 | if (!static_branch_likely(&blake2s_use_ssse3) || !may_use_simd()) { |
35 | blake2s_compress_generic(state, block, nblocks, inc); |
36 | return; |
37 | } |
38 | |
39 | do { |
40 | const size_t blocks = min_t(size_t, nblocks, |
41 | SZ_4K / BLAKE2S_BLOCK_SIZE); |
42 | |
43 | kernel_fpu_begin(); |
44 | if (IS_ENABLED(CONFIG_AS_AVX512) && |
45 | static_branch_likely(&blake2s_use_avx512)) |
46 | blake2s_compress_avx512(state, block, nblocks: blocks, inc); |
47 | else |
48 | blake2s_compress_ssse3(state, block, nblocks: blocks, inc); |
49 | kernel_fpu_end(); |
50 | |
51 | nblocks -= blocks; |
52 | block += blocks * BLAKE2S_BLOCK_SIZE; |
53 | } while (nblocks); |
54 | } |
55 | EXPORT_SYMBOL(blake2s_compress); |
56 | |
57 | static int __init blake2s_mod_init(void) |
58 | { |
59 | if (boot_cpu_has(X86_FEATURE_SSSE3)) |
60 | static_branch_enable(&blake2s_use_ssse3); |
61 | |
62 | if (IS_ENABLED(CONFIG_AS_AVX512) && |
63 | boot_cpu_has(X86_FEATURE_AVX) && |
64 | boot_cpu_has(X86_FEATURE_AVX2) && |
65 | boot_cpu_has(X86_FEATURE_AVX512F) && |
66 | boot_cpu_has(X86_FEATURE_AVX512VL) && |
67 | cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM | |
68 | XFEATURE_MASK_AVX512, NULL)) |
69 | static_branch_enable(&blake2s_use_avx512); |
70 | |
71 | return 0; |
72 | } |
73 | |
74 | subsys_initcall(blake2s_mod_init); |
75 | |