| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
|---|
| 2 | /* |
|---|
| 3 | * x86-optimized SHA-512 block function |
|---|
| 4 | * |
|---|
| 5 | * Copyright 2025 Google LLC |
|---|
| 6 | */ |
|---|
| 7 | #include <asm/fpu/api.h> |
|---|
| 8 | #include <linux/static_call.h> |
|---|
| 9 | |
|---|
| 10 | DEFINE_STATIC_CALL(sha512_blocks_x86, sha512_blocks_generic); |
|---|
| 11 | |
|---|
| 12 | #define DEFINE_X86_SHA512_FN(c_fn, asm_fn) \ |
|---|
| 13 | asmlinkage void asm_fn(struct sha512_block_state *state, \ |
|---|
| 14 | const u8 *data, size_t nblocks); \ |
|---|
| 15 | static void c_fn(struct sha512_block_state *state, const u8 *data, \ |
|---|
| 16 | size_t nblocks) \ |
|---|
| 17 | { \ |
|---|
| 18 | if (likely(irq_fpu_usable())) { \ |
|---|
| 19 | kernel_fpu_begin(); \ |
|---|
| 20 | asm_fn(state, data, nblocks); \ |
|---|
| 21 | kernel_fpu_end(); \ |
|---|
| 22 | } else { \ |
|---|
| 23 | sha512_blocks_generic(state, data, nblocks); \ |
|---|
| 24 | } \ |
|---|
| 25 | } |
|---|
| 26 | |
|---|
| 27 | DEFINE_X86_SHA512_FN(sha512_blocks_ssse3, sha512_transform_ssse3); |
|---|
| 28 | DEFINE_X86_SHA512_FN(sha512_blocks_avx, sha512_transform_avx); |
|---|
| 29 | DEFINE_X86_SHA512_FN(sha512_blocks_avx2, sha512_transform_rorx); |
|---|
| 30 | |
|---|
| 31 | static void sha512_blocks(struct sha512_block_state *state, |
|---|
| 32 | const u8 *data, size_t nblocks) |
|---|
| 33 | { |
|---|
| 34 | static_call(sha512_blocks_x86)(state, data, nblocks); |
|---|
| 35 | } |
|---|
| 36 | |
|---|
| 37 | #define sha512_mod_init_arch sha512_mod_init_arch |
|---|
| 38 | static void sha512_mod_init_arch(void) |
|---|
| 39 | { |
|---|
| 40 | if (cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL) && |
|---|
| 41 | boot_cpu_has(X86_FEATURE_AVX)) { |
|---|
| 42 | if (boot_cpu_has(X86_FEATURE_AVX2) && |
|---|
| 43 | boot_cpu_has(X86_FEATURE_BMI2)) |
|---|
| 44 | static_call_update(sha512_blocks_x86, |
|---|
| 45 | sha512_blocks_avx2); |
|---|
| 46 | else |
|---|
| 47 | static_call_update(sha512_blocks_x86, |
|---|
| 48 | sha512_blocks_avx); |
|---|
| 49 | } else if (boot_cpu_has(X86_FEATURE_SSSE3)) { |
|---|
| 50 | static_call_update(sha512_blocks_x86, sha512_blocks_ssse3); |
|---|
| 51 | } |
|---|
| 52 | } |
|---|
| 53 | |
|---|
