1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Glue code for POLYVAL using ARMv8 Crypto Extensions |
4 | * |
5 | * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> |
6 | * Copyright (c) 2009 Intel Corp. |
7 | * Author: Huang Ying <ying.huang@intel.com> |
8 | * Copyright 2021 Google LLC |
9 | */ |
10 | |
11 | /* |
12 | * Glue code based on ghash-clmulni-intel_glue.c. |
13 | * |
14 | * This implementation of POLYVAL uses montgomery multiplication accelerated by |
15 | * ARMv8 Crypto Extensions instructions to implement the finite field operations. |
16 | */ |
17 | |
18 | #include <crypto/algapi.h> |
19 | #include <crypto/internal/hash.h> |
20 | #include <crypto/internal/simd.h> |
21 | #include <crypto/polyval.h> |
22 | #include <linux/crypto.h> |
23 | #include <linux/init.h> |
24 | #include <linux/kernel.h> |
25 | #include <linux/module.h> |
26 | #include <linux/cpufeature.h> |
27 | #include <asm/neon.h> |
28 | #include <asm/simd.h> |
29 | |
30 | #define NUM_KEY_POWERS 8 |
31 | |
32 | struct polyval_tfm_ctx { |
33 | /* |
34 | * These powers must be in the order h^8, ..., h^1. |
35 | */ |
36 | u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE]; |
37 | }; |
38 | |
39 | struct polyval_desc_ctx { |
40 | u8 buffer[POLYVAL_BLOCK_SIZE]; |
41 | u32 bytes; |
42 | }; |
43 | |
44 | asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys, |
45 | const u8 *in, size_t nblocks, u8 *accumulator); |
46 | asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2); |
47 | |
48 | static void internal_polyval_update(const struct polyval_tfm_ctx *keys, |
49 | const u8 *in, size_t nblocks, u8 *accumulator) |
50 | { |
51 | if (likely(crypto_simd_usable())) { |
52 | kernel_neon_begin(); |
53 | pmull_polyval_update(keys, in, nblocks, accumulator); |
54 | kernel_neon_end(); |
55 | } else { |
56 | polyval_update_non4k(key: keys->key_powers[NUM_KEY_POWERS-1], in, |
57 | nblocks, accumulator); |
58 | } |
59 | } |
60 | |
61 | static void internal_polyval_mul(u8 *op1, const u8 *op2) |
62 | { |
63 | if (likely(crypto_simd_usable())) { |
64 | kernel_neon_begin(); |
65 | pmull_polyval_mul(op1, op2); |
66 | kernel_neon_end(); |
67 | } else { |
68 | polyval_mul_non4k(op1, op2); |
69 | } |
70 | } |
71 | |
72 | static int polyval_arm64_setkey(struct crypto_shash *tfm, |
73 | const u8 *key, unsigned int keylen) |
74 | { |
75 | struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm); |
76 | int i; |
77 | |
78 | if (keylen != POLYVAL_BLOCK_SIZE) |
79 | return -EINVAL; |
80 | |
81 | memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE); |
82 | |
83 | for (i = NUM_KEY_POWERS-2; i >= 0; i--) { |
84 | memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE); |
85 | internal_polyval_mul(op1: tctx->key_powers[i], |
86 | op2: tctx->key_powers[i+1]); |
87 | } |
88 | |
89 | return 0; |
90 | } |
91 | |
92 | static int polyval_arm64_init(struct shash_desc *desc) |
93 | { |
94 | struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); |
95 | |
96 | memset(dctx, 0, sizeof(*dctx)); |
97 | |
98 | return 0; |
99 | } |
100 | |
101 | static int polyval_arm64_update(struct shash_desc *desc, |
102 | const u8 *src, unsigned int srclen) |
103 | { |
104 | struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); |
105 | const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm); |
106 | u8 *pos; |
107 | unsigned int nblocks; |
108 | unsigned int n; |
109 | |
110 | if (dctx->bytes) { |
111 | n = min(srclen, dctx->bytes); |
112 | pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes; |
113 | |
114 | dctx->bytes -= n; |
115 | srclen -= n; |
116 | |
117 | while (n--) |
118 | *pos++ ^= *src++; |
119 | |
120 | if (!dctx->bytes) |
121 | internal_polyval_mul(op1: dctx->buffer, |
122 | op2: tctx->key_powers[NUM_KEY_POWERS-1]); |
123 | } |
124 | |
125 | while (srclen >= POLYVAL_BLOCK_SIZE) { |
126 | /* allow rescheduling every 4K bytes */ |
127 | nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE; |
128 | internal_polyval_update(keys: tctx, in: src, nblocks, accumulator: dctx->buffer); |
129 | srclen -= nblocks * POLYVAL_BLOCK_SIZE; |
130 | src += nblocks * POLYVAL_BLOCK_SIZE; |
131 | } |
132 | |
133 | if (srclen) { |
134 | dctx->bytes = POLYVAL_BLOCK_SIZE - srclen; |
135 | pos = dctx->buffer; |
136 | while (srclen--) |
137 | *pos++ ^= *src++; |
138 | } |
139 | |
140 | return 0; |
141 | } |
142 | |
143 | static int polyval_arm64_final(struct shash_desc *desc, u8 *dst) |
144 | { |
145 | struct polyval_desc_ctx *dctx = shash_desc_ctx(desc); |
146 | const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm: desc->tfm); |
147 | |
148 | if (dctx->bytes) { |
149 | internal_polyval_mul(op1: dctx->buffer, |
150 | op2: tctx->key_powers[NUM_KEY_POWERS-1]); |
151 | } |
152 | |
153 | memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE); |
154 | |
155 | return 0; |
156 | } |
157 | |
158 | static struct shash_alg polyval_alg = { |
159 | .digestsize = POLYVAL_DIGEST_SIZE, |
160 | .init = polyval_arm64_init, |
161 | .update = polyval_arm64_update, |
162 | .final = polyval_arm64_final, |
163 | .setkey = polyval_arm64_setkey, |
164 | .descsize = sizeof(struct polyval_desc_ctx), |
165 | .base = { |
166 | .cra_name = "polyval" , |
167 | .cra_driver_name = "polyval-ce" , |
168 | .cra_priority = 200, |
169 | .cra_blocksize = POLYVAL_BLOCK_SIZE, |
170 | .cra_ctxsize = sizeof(struct polyval_tfm_ctx), |
171 | .cra_module = THIS_MODULE, |
172 | }, |
173 | }; |
174 | |
175 | static int __init polyval_ce_mod_init(void) |
176 | { |
177 | return crypto_register_shash(alg: &polyval_alg); |
178 | } |
179 | |
180 | static void __exit polyval_ce_mod_exit(void) |
181 | { |
182 | crypto_unregister_shash(alg: &polyval_alg); |
183 | } |
184 | |
185 | module_cpu_feature_match(PMULL, polyval_ce_mod_init) |
186 | module_exit(polyval_ce_mod_exit); |
187 | |
188 | MODULE_LICENSE("GPL" ); |
189 | MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions" ); |
190 | MODULE_ALIAS_CRYPTO("polyval" ); |
191 | MODULE_ALIAS_CRYPTO("polyval-ce" ); |
192 | |