1// SPDX-License-Identifier: GPL-2.0
2/*
3 * GHASH routines supporting VMX instructions on the Power 8
4 *
5 * Copyright (C) 2015, 2019 International Business Machines Inc.
6 *
7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
8 *
9 * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
10 * mechanism. The new approach is based on arm64 code, which is:
11 * Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
12 */
13
14#include "aesp8-ppc.h"
15#include <asm/switch_to.h>
16#include <crypto/aes.h>
17#include <crypto/gf128mul.h>
18#include <crypto/ghash.h>
19#include <crypto/internal/hash.h>
20#include <crypto/internal/simd.h>
21#include <linux/err.h>
22#include <linux/kernel.h>
23#include <linux/module.h>
24#include <linux/string.h>
25#include <linux/uaccess.h>
26
27void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
28void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
29void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
30 const u8 *in, size_t len);
31
32struct p8_ghash_ctx {
33 /* key used by vector asm */
34 u128 htable[16];
35 /* key used by software fallback */
36 be128 key;
37};
38
39struct p8_ghash_desc_ctx {
40 u64 shash[2];
41};
42
43static int p8_ghash_init(struct shash_desc *desc)
44{
45 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
46
47 memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
48 return 0;
49}
50
51static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
52 unsigned int keylen)
53{
54 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm));
55
56 if (keylen != GHASH_BLOCK_SIZE)
57 return -EINVAL;
58
59 preempt_disable();
60 pagefault_disable();
61 enable_kernel_vsx();
62 gcm_init_p8(htable: ctx->htable, Xi: (const u64 *) key);
63 disable_kernel_vsx();
64 pagefault_enable();
65 preempt_enable();
66
67 memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
68
69 return 0;
70}
71
72static inline void __ghash_block(struct p8_ghash_ctx *ctx,
73 struct p8_ghash_desc_ctx *dctx,
74 const u8 *src)
75{
76 if (crypto_simd_usable()) {
77 preempt_disable();
78 pagefault_disable();
79 enable_kernel_vsx();
80 gcm_ghash_p8(Xi: dctx->shash, htable: ctx->htable, in: src, GHASH_BLOCK_SIZE);
81 disable_kernel_vsx();
82 pagefault_enable();
83 preempt_enable();
84 } else {
85 crypto_xor(dst: (u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
86 gf128mul_lle(a: (be128 *)dctx->shash, b: &ctx->key);
87 }
88}
89
90static inline int __ghash_blocks(struct p8_ghash_ctx *ctx,
91 struct p8_ghash_desc_ctx *dctx,
92 const u8 *src, unsigned int srclen)
93{
94 int remain = srclen - round_down(srclen, GHASH_BLOCK_SIZE);
95
96 srclen -= remain;
97 if (crypto_simd_usable()) {
98 preempt_disable();
99 pagefault_disable();
100 enable_kernel_vsx();
101 gcm_ghash_p8(Xi: dctx->shash, htable: ctx->htable,
102 in: src, len: srclen);
103 disable_kernel_vsx();
104 pagefault_enable();
105 preempt_enable();
106 } else {
107 do {
108 crypto_xor(dst: (u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
109 gf128mul_lle(a: (be128 *)dctx->shash, b: &ctx->key);
110 srclen -= GHASH_BLOCK_SIZE;
111 src += GHASH_BLOCK_SIZE;
112 } while (srclen);
113 }
114
115 return remain;
116}
117
118static int p8_ghash_update(struct shash_desc *desc,
119 const u8 *src, unsigned int srclen)
120{
121 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm: desc->tfm));
122 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
123
124 return __ghash_blocks(ctx, dctx, src, srclen);
125}
126
127static int p8_ghash_finup(struct shash_desc *desc, const u8 *src,
128 unsigned int len, u8 *out)
129{
130 struct p8_ghash_ctx *ctx = crypto_tfm_ctx(tfm: crypto_shash_tfm(tfm: desc->tfm));
131 struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
132
133 if (len) {
134 u8 buf[GHASH_BLOCK_SIZE] = {};
135
136 memcpy(buf, src, len);
137 __ghash_block(ctx, dctx, src: buf);
138 memzero_explicit(s: buf, count: sizeof(buf));
139 }
140 memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
141 return 0;
142}
143
144struct shash_alg p8_ghash_alg = {
145 .digestsize = GHASH_DIGEST_SIZE,
146 .init = p8_ghash_init,
147 .update = p8_ghash_update,
148 .finup = p8_ghash_finup,
149 .setkey = p8_ghash_setkey,
150 .descsize = sizeof(struct p8_ghash_desc_ctx),
151 .base = {
152 .cra_name = "ghash",
153 .cra_driver_name = "p8_ghash",
154 .cra_priority = 1000,
155 .cra_flags = CRYPTO_AHASH_ALG_BLOCK_ONLY,
156 .cra_blocksize = GHASH_BLOCK_SIZE,
157 .cra_ctxsize = sizeof(struct p8_ghash_ctx),
158 .cra_module = THIS_MODULE,
159 },
160};
161

source code of linux/arch/powerpc/crypto/ghash.c