1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Glue code for POLYVAL using PCMULQDQ-NI
4 *
5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 * Copyright (c) 2009 Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 * Copyright 2021 Google LLC
9 */
10
11/*
12 * Glue code based on ghash-clmulni-intel_glue.c.
13 *
14 * This implementation of POLYVAL uses montgomery multiplication
15 * accelerated by PCLMULQDQ-NI to implement the finite field
16 * operations.
17 */
18
19#include <crypto/algapi.h>
20#include <crypto/internal/hash.h>
21#include <crypto/internal/simd.h>
22#include <crypto/polyval.h>
23#include <linux/crypto.h>
24#include <linux/init.h>
25#include <linux/kernel.h>
26#include <linux/module.h>
27#include <asm/cpu_device_id.h>
28#include <asm/simd.h>
29
30#define POLYVAL_ALIGN 16
31#define POLYVAL_ALIGN_ATTR __aligned(POLYVAL_ALIGN)
32#define POLYVAL_ALIGN_EXTRA ((POLYVAL_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
33#define POLYVAL_CTX_SIZE (sizeof(struct polyval_tfm_ctx) + POLYVAL_ALIGN_EXTRA)
34#define NUM_KEY_POWERS 8
35
36struct polyval_tfm_ctx {
37 /*
38 * These powers must be in the order h^8, ..., h^1.
39 */
40 u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE] POLYVAL_ALIGN_ATTR;
41};
42
43struct polyval_desc_ctx {
44 u8 buffer[POLYVAL_BLOCK_SIZE];
45 u32 bytes;
46};
47
48asmlinkage void clmul_polyval_update(const struct polyval_tfm_ctx *keys,
49 const u8 *in, size_t nblocks, u8 *accumulator);
50asmlinkage void clmul_polyval_mul(u8 *op1, const u8 *op2);
51
52static inline struct polyval_tfm_ctx *polyval_tfm_ctx(struct crypto_shash *tfm)
53{
54 return PTR_ALIGN(crypto_shash_ctx(tfm), POLYVAL_ALIGN);
55}
56
57static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
58 const u8 *in, size_t nblocks, u8 *accumulator)
59{
60 if (likely(crypto_simd_usable())) {
61 kernel_fpu_begin();
62 clmul_polyval_update(keys, in, nblocks, accumulator);
63 kernel_fpu_end();
64 } else {
65 polyval_update_non4k(key: keys->key_powers[NUM_KEY_POWERS-1], in,
66 nblocks, accumulator);
67 }
68}
69
70static void internal_polyval_mul(u8 *op1, const u8 *op2)
71{
72 if (likely(crypto_simd_usable())) {
73 kernel_fpu_begin();
74 clmul_polyval_mul(op1, op2);
75 kernel_fpu_end();
76 } else {
77 polyval_mul_non4k(op1, op2);
78 }
79}
80
81static int polyval_x86_setkey(struct crypto_shash *tfm,
82 const u8 *key, unsigned int keylen)
83{
84 struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm);
85 int i;
86
87 if (keylen != POLYVAL_BLOCK_SIZE)
88 return -EINVAL;
89
90 memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
91
92 for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
93 memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
94 internal_polyval_mul(op1: tctx->key_powers[i],
95 op2: tctx->key_powers[i+1]);
96 }
97
98 return 0;
99}
100
101static int polyval_x86_init(struct shash_desc *desc)
102{
103 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
104
105 memset(dctx, 0, sizeof(*dctx));
106
107 return 0;
108}
109
110static int polyval_x86_update(struct shash_desc *desc,
111 const u8 *src, unsigned int srclen)
112{
113 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
114 const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm: desc->tfm);
115 u8 *pos;
116 unsigned int nblocks;
117 unsigned int n;
118
119 if (dctx->bytes) {
120 n = min(srclen, dctx->bytes);
121 pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
122
123 dctx->bytes -= n;
124 srclen -= n;
125
126 while (n--)
127 *pos++ ^= *src++;
128
129 if (!dctx->bytes)
130 internal_polyval_mul(op1: dctx->buffer,
131 op2: tctx->key_powers[NUM_KEY_POWERS-1]);
132 }
133
134 while (srclen >= POLYVAL_BLOCK_SIZE) {
135 /* Allow rescheduling every 4K bytes. */
136 nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
137 internal_polyval_update(keys: tctx, in: src, nblocks, accumulator: dctx->buffer);
138 srclen -= nblocks * POLYVAL_BLOCK_SIZE;
139 src += nblocks * POLYVAL_BLOCK_SIZE;
140 }
141
142 if (srclen) {
143 dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
144 pos = dctx->buffer;
145 while (srclen--)
146 *pos++ ^= *src++;
147 }
148
149 return 0;
150}
151
152static int polyval_x86_final(struct shash_desc *desc, u8 *dst)
153{
154 struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
155 const struct polyval_tfm_ctx *tctx = polyval_tfm_ctx(tfm: desc->tfm);
156
157 if (dctx->bytes) {
158 internal_polyval_mul(op1: dctx->buffer,
159 op2: tctx->key_powers[NUM_KEY_POWERS-1]);
160 }
161
162 memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
163
164 return 0;
165}
166
167static struct shash_alg polyval_alg = {
168 .digestsize = POLYVAL_DIGEST_SIZE,
169 .init = polyval_x86_init,
170 .update = polyval_x86_update,
171 .final = polyval_x86_final,
172 .setkey = polyval_x86_setkey,
173 .descsize = sizeof(struct polyval_desc_ctx),
174 .base = {
175 .cra_name = "polyval",
176 .cra_driver_name = "polyval-clmulni",
177 .cra_priority = 200,
178 .cra_blocksize = POLYVAL_BLOCK_SIZE,
179 .cra_ctxsize = POLYVAL_CTX_SIZE,
180 .cra_module = THIS_MODULE,
181 },
182};
183
184__maybe_unused static const struct x86_cpu_id pcmul_cpu_id[] = {
185 X86_MATCH_FEATURE(X86_FEATURE_PCLMULQDQ, NULL),
186 {}
187};
188MODULE_DEVICE_TABLE(x86cpu, pcmul_cpu_id);
189
190static int __init polyval_clmulni_mod_init(void)
191{
192 if (!x86_match_cpu(match: pcmul_cpu_id))
193 return -ENODEV;
194
195 if (!boot_cpu_has(X86_FEATURE_AVX))
196 return -ENODEV;
197
198 return crypto_register_shash(alg: &polyval_alg);
199}
200
201static void __exit polyval_clmulni_mod_exit(void)
202{
203 crypto_unregister_shash(alg: &polyval_alg);
204}
205
206module_init(polyval_clmulni_mod_init);
207module_exit(polyval_clmulni_mod_exit);
208
209MODULE_LICENSE("GPL");
210MODULE_DESCRIPTION("POLYVAL hash function accelerated by PCLMULQDQ-NI");
211MODULE_ALIAS_CRYPTO("polyval");
212MODULE_ALIAS_CRYPTO("polyval-clmulni");
213

source code of linux/arch/x86/crypto/polyval-clmulni_glue.c