Warning: This file is not a C or C++ file. It does not have highlighting.
| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
|---|---|
| 2 | /* |
| 3 | * SHA-3 optimized using the CP Assist for Cryptographic Functions (CPACF) |
| 4 | * |
| 5 | * Copyright 2025 Google LLC |
| 6 | */ |
| 7 | #include <asm/cpacf.h> |
| 8 | #include <linux/cpufeature.h> |
| 9 | |
| 10 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3); |
| 11 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_sha3_init_optim); |
| 12 | |
| 13 | static void sha3_absorb_blocks(struct sha3_state *state, const u8 *data, |
| 14 | size_t nblocks, size_t block_size) |
| 15 | { |
| 16 | if (static_branch_likely(&have_sha3)) { |
| 17 | /* |
| 18 | * Note that KIMD assumes little-endian order of the state |
| 19 | * words. sha3_state already uses that order, though, so |
| 20 | * there's no need for a byteswap. |
| 21 | */ |
| 22 | switch (block_size) { |
| 23 | case SHA3_224_BLOCK_SIZE: |
| 24 | cpacf_kimd(CPACF_KIMD_SHA3_224, state, |
| 25 | data, nblocks * block_size); |
| 26 | return; |
| 27 | case SHA3_256_BLOCK_SIZE: |
| 28 | /* |
| 29 | * This case handles both SHA3-256 and SHAKE256, since |
| 30 | * they have the same block size. |
| 31 | */ |
| 32 | cpacf_kimd(CPACF_KIMD_SHA3_256, state, |
| 33 | data, nblocks * block_size); |
| 34 | return; |
| 35 | case SHA3_384_BLOCK_SIZE: |
| 36 | cpacf_kimd(CPACF_KIMD_SHA3_384, state, |
| 37 | data, nblocks * block_size); |
| 38 | return; |
| 39 | case SHA3_512_BLOCK_SIZE: |
| 40 | cpacf_kimd(CPACF_KIMD_SHA3_512, state, |
| 41 | data, nblocks * block_size); |
| 42 | return; |
| 43 | } |
| 44 | } |
| 45 | sha3_absorb_blocks_generic(state, data, nblocks, block_size); |
| 46 | } |
| 47 | |
| 48 | static void sha3_keccakf(struct sha3_state *state) |
| 49 | { |
| 50 | if (static_branch_likely(&have_sha3)) { |
| 51 | /* |
| 52 | * Passing zeroes into any of CPACF_KIMD_SHA3_* gives the plain |
| 53 | * Keccak-f permutation, which is what we want here. Use |
| 54 | * SHA3-512 since it has the smallest block size. |
| 55 | */ |
| 56 | static const u8 zeroes[SHA3_512_BLOCK_SIZE]; |
| 57 | |
| 58 | cpacf_kimd(CPACF_KIMD_SHA3_512, state, zeroes, sizeof(zeroes)); |
| 59 | } else { |
| 60 | sha3_keccakf_generic(state); |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | static inline bool s390_sha3(int func, const u8 *in, size_t in_len, |
| 65 | u8 *out, size_t out_len) |
| 66 | { |
| 67 | struct sha3_state state; |
| 68 | |
| 69 | if (!static_branch_likely(&have_sha3)) |
| 70 | return false; |
| 71 | |
| 72 | if (static_branch_likely(&have_sha3_init_optim)) |
| 73 | func |= CPACF_KLMD_NIP | CPACF_KLMD_DUFOP; |
| 74 | else |
| 75 | memset(&state, 0, sizeof(state)); |
| 76 | |
| 77 | cpacf_klmd(func, &state, in, in_len); |
| 78 | |
| 79 | if (static_branch_likely(&have_sha3_init_optim)) |
| 80 | kmsan_unpoison_memory(&state, out_len); |
| 81 | |
| 82 | memcpy(out, &state, out_len); |
| 83 | memzero_explicit(&state, sizeof(state)); |
| 84 | return true; |
| 85 | } |
| 86 | |
| 87 | #define sha3_224_arch sha3_224_arch |
| 88 | static bool sha3_224_arch(const u8 *in, size_t in_len, |
| 89 | u8 out[SHA3_224_DIGEST_SIZE]) |
| 90 | { |
| 91 | return s390_sha3(CPACF_KLMD_SHA3_224, in, in_len, |
| 92 | out, SHA3_224_DIGEST_SIZE); |
| 93 | } |
| 94 | |
| 95 | #define sha3_256_arch sha3_256_arch |
| 96 | static bool sha3_256_arch(const u8 *in, size_t in_len, |
| 97 | u8 out[SHA3_256_DIGEST_SIZE]) |
| 98 | { |
| 99 | return s390_sha3(CPACF_KLMD_SHA3_256, in, in_len, |
| 100 | out, SHA3_256_DIGEST_SIZE); |
| 101 | } |
| 102 | |
| 103 | #define sha3_384_arch sha3_384_arch |
| 104 | static bool sha3_384_arch(const u8 *in, size_t in_len, |
| 105 | u8 out[SHA3_384_DIGEST_SIZE]) |
| 106 | { |
| 107 | return s390_sha3(CPACF_KLMD_SHA3_384, in, in_len, |
| 108 | out, SHA3_384_DIGEST_SIZE); |
| 109 | } |
| 110 | |
| 111 | #define sha3_512_arch sha3_512_arch |
| 112 | static bool sha3_512_arch(const u8 *in, size_t in_len, |
| 113 | u8 out[SHA3_512_DIGEST_SIZE]) |
| 114 | { |
| 115 | return s390_sha3(CPACF_KLMD_SHA3_512, in, in_len, |
| 116 | out, SHA3_512_DIGEST_SIZE); |
| 117 | } |
| 118 | |
| 119 | #define sha3_mod_init_arch sha3_mod_init_arch |
| 120 | static void sha3_mod_init_arch(void) |
| 121 | { |
| 122 | int num_present = 0; |
| 123 | int num_possible = 0; |
| 124 | |
| 125 | if (!cpu_have_feature(S390_CPU_FEATURE_MSA)) |
| 126 | return; |
| 127 | /* |
| 128 | * Since all the SHA-3 functions are in Message-Security-Assist |
| 129 | * Extension 6, just treat them as all or nothing. This way we need |
| 130 | * only one static_key. |
| 131 | */ |
| 132 | #define QUERY(opcode, func) \ |
| 133 | ({ num_present += !!cpacf_query_func(opcode, func); num_possible++; }) |
| 134 | QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_224); |
| 135 | QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_256); |
| 136 | QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_384); |
| 137 | QUERY(CPACF_KIMD, CPACF_KIMD_SHA3_512); |
| 138 | QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_224); |
| 139 | QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_256); |
| 140 | QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_384); |
| 141 | QUERY(CPACF_KLMD, CPACF_KLMD_SHA3_512); |
| 142 | #undef QUERY |
| 143 | |
| 144 | if (num_present == num_possible) { |
| 145 | static_branch_enable(&have_sha3); |
| 146 | if (test_facility(86)) |
| 147 | static_branch_enable(&have_sha3_init_optim); |
| 148 | } else if (num_present != 0) { |
| 149 | pr_warn("Unsupported combination of SHA-3 facilities\n"); |
| 150 | } |
| 151 | } |
| 152 |
Warning: This file is not a C or C++ file. It does not have highlighting.