1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * crc32.c - CRC32 and CRC32C using LoongArch crc* instructions |
4 | * |
5 | * Module based on mips/crypto/crc32-mips.c |
6 | * |
7 | * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org> |
8 | * Copyright (C) 2018 MIPS Tech, LLC |
9 | * Copyright (C) 2020-2023 Loongson Technology Corporation Limited |
10 | */ |
11 | |
12 | #include <linux/module.h> |
13 | #include <crypto/internal/hash.h> |
14 | |
15 | #include <asm/cpu-features.h> |
16 | #include <asm/unaligned.h> |
17 | |
18 | #define _CRC32(crc, value, size, type) \ |
19 | do { \ |
20 | __asm__ __volatile__( \ |
21 | #type ".w." #size ".w" " %0, %1, %0\n\t"\ |
22 | : "+r" (crc) \ |
23 | : "r" (value) \ |
24 | : "memory"); \ |
25 | } while (0) |
26 | |
27 | #define CRC32(crc, value, size) _CRC32(crc, value, size, crc) |
28 | #define CRC32C(crc, value, size) _CRC32(crc, value, size, crcc) |
29 | |
30 | static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len) |
31 | { |
32 | u32 crc = crc_; |
33 | |
34 | while (len >= sizeof(u64)) { |
35 | u64 value = get_unaligned_le64(p); |
36 | |
37 | CRC32(crc, value, d); |
38 | p += sizeof(u64); |
39 | len -= sizeof(u64); |
40 | } |
41 | |
42 | if (len & sizeof(u32)) { |
43 | u32 value = get_unaligned_le32(p); |
44 | |
45 | CRC32(crc, value, w); |
46 | p += sizeof(u32); |
47 | } |
48 | |
49 | if (len & sizeof(u16)) { |
50 | u16 value = get_unaligned_le16(p); |
51 | |
52 | CRC32(crc, value, h); |
53 | p += sizeof(u16); |
54 | } |
55 | |
56 | if (len & sizeof(u8)) { |
57 | u8 value = *p++; |
58 | |
59 | CRC32(crc, value, b); |
60 | } |
61 | |
62 | return crc; |
63 | } |
64 | |
65 | static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len) |
66 | { |
67 | u32 crc = crc_; |
68 | |
69 | while (len >= sizeof(u64)) { |
70 | u64 value = get_unaligned_le64(p); |
71 | |
72 | CRC32C(crc, value, d); |
73 | p += sizeof(u64); |
74 | len -= sizeof(u64); |
75 | } |
76 | |
77 | if (len & sizeof(u32)) { |
78 | u32 value = get_unaligned_le32(p); |
79 | |
80 | CRC32C(crc, value, w); |
81 | p += sizeof(u32); |
82 | } |
83 | |
84 | if (len & sizeof(u16)) { |
85 | u16 value = get_unaligned_le16(p); |
86 | |
87 | CRC32C(crc, value, h); |
88 | p += sizeof(u16); |
89 | } |
90 | |
91 | if (len & sizeof(u8)) { |
92 | u8 value = *p++; |
93 | |
94 | CRC32C(crc, value, b); |
95 | } |
96 | |
97 | return crc; |
98 | } |
99 | |
100 | #define CHKSUM_BLOCK_SIZE 1 |
101 | #define CHKSUM_DIGEST_SIZE 4 |
102 | |
103 | struct chksum_ctx { |
104 | u32 key; |
105 | }; |
106 | |
107 | struct chksum_desc_ctx { |
108 | u32 crc; |
109 | }; |
110 | |
111 | static int chksum_init(struct shash_desc *desc) |
112 | { |
113 | struct chksum_ctx *mctx = crypto_shash_ctx(tfm: desc->tfm); |
114 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
115 | |
116 | ctx->crc = mctx->key; |
117 | |
118 | return 0; |
119 | } |
120 | |
121 | /* |
122 | * Setting the seed allows arbitrary accumulators and flexible XOR policy |
123 | * If your algorithm starts with ~0, then XOR with ~0 before you set the seed. |
124 | */ |
125 | static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) |
126 | { |
127 | struct chksum_ctx *mctx = crypto_shash_ctx(tfm); |
128 | |
129 | if (keylen != sizeof(mctx->key)) |
130 | return -EINVAL; |
131 | |
132 | mctx->key = get_unaligned_le32(p: key); |
133 | |
134 | return 0; |
135 | } |
136 | |
137 | static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length) |
138 | { |
139 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
140 | |
141 | ctx->crc = crc32_loongarch_hw(crc_: ctx->crc, p: data, len: length); |
142 | return 0; |
143 | } |
144 | |
145 | static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length) |
146 | { |
147 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
148 | |
149 | ctx->crc = crc32c_loongarch_hw(crc_: ctx->crc, p: data, len: length); |
150 | return 0; |
151 | } |
152 | |
153 | static int chksum_final(struct shash_desc *desc, u8 *out) |
154 | { |
155 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
156 | |
157 | put_unaligned_le32(val: ctx->crc, p: out); |
158 | return 0; |
159 | } |
160 | |
161 | static int chksumc_final(struct shash_desc *desc, u8 *out) |
162 | { |
163 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
164 | |
165 | put_unaligned_le32(val: ~ctx->crc, p: out); |
166 | return 0; |
167 | } |
168 | |
169 | static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) |
170 | { |
171 | put_unaligned_le32(val: crc32_loongarch_hw(crc_: crc, p: data, len), p: out); |
172 | return 0; |
173 | } |
174 | |
175 | static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out) |
176 | { |
177 | put_unaligned_le32(val: ~crc32c_loongarch_hw(crc_: crc, p: data, len), p: out); |
178 | return 0; |
179 | } |
180 | |
181 | static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) |
182 | { |
183 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
184 | |
185 | return __chksum_finup(crc: ctx->crc, data, len, out); |
186 | } |
187 | |
188 | static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) |
189 | { |
190 | struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); |
191 | |
192 | return __chksumc_finup(crc: ctx->crc, data, len, out); |
193 | } |
194 | |
195 | static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) |
196 | { |
197 | struct chksum_ctx *mctx = crypto_shash_ctx(tfm: desc->tfm); |
198 | |
199 | return __chksum_finup(crc: mctx->key, data, len: length, out); |
200 | } |
201 | |
202 | static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) |
203 | { |
204 | struct chksum_ctx *mctx = crypto_shash_ctx(tfm: desc->tfm); |
205 | |
206 | return __chksumc_finup(crc: mctx->key, data, len: length, out); |
207 | } |
208 | |
209 | static int chksum_cra_init(struct crypto_tfm *tfm) |
210 | { |
211 | struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); |
212 | |
213 | mctx->key = 0; |
214 | return 0; |
215 | } |
216 | |
217 | static int chksumc_cra_init(struct crypto_tfm *tfm) |
218 | { |
219 | struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); |
220 | |
221 | mctx->key = ~0; |
222 | return 0; |
223 | } |
224 | |
225 | static struct shash_alg crc32_alg = { |
226 | .digestsize = CHKSUM_DIGEST_SIZE, |
227 | .setkey = chksum_setkey, |
228 | .init = chksum_init, |
229 | .update = chksum_update, |
230 | .final = chksum_final, |
231 | .finup = chksum_finup, |
232 | .digest = chksum_digest, |
233 | .descsize = sizeof(struct chksum_desc_ctx), |
234 | .base = { |
235 | .cra_name = "crc32" , |
236 | .cra_driver_name = "crc32-loongarch" , |
237 | .cra_priority = 300, |
238 | .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, |
239 | .cra_blocksize = CHKSUM_BLOCK_SIZE, |
240 | .cra_ctxsize = sizeof(struct chksum_ctx), |
241 | .cra_module = THIS_MODULE, |
242 | .cra_init = chksum_cra_init, |
243 | } |
244 | }; |
245 | |
246 | static struct shash_alg crc32c_alg = { |
247 | .digestsize = CHKSUM_DIGEST_SIZE, |
248 | .setkey = chksum_setkey, |
249 | .init = chksum_init, |
250 | .update = chksumc_update, |
251 | .final = chksumc_final, |
252 | .finup = chksumc_finup, |
253 | .digest = chksumc_digest, |
254 | .descsize = sizeof(struct chksum_desc_ctx), |
255 | .base = { |
256 | .cra_name = "crc32c" , |
257 | .cra_driver_name = "crc32c-loongarch" , |
258 | .cra_priority = 300, |
259 | .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, |
260 | .cra_blocksize = CHKSUM_BLOCK_SIZE, |
261 | .cra_ctxsize = sizeof(struct chksum_ctx), |
262 | .cra_module = THIS_MODULE, |
263 | .cra_init = chksumc_cra_init, |
264 | } |
265 | }; |
266 | |
267 | static int __init crc32_mod_init(void) |
268 | { |
269 | int err; |
270 | |
271 | if (!cpu_has(CPU_FEATURE_CRC32)) |
272 | return 0; |
273 | |
274 | err = crypto_register_shash(alg: &crc32_alg); |
275 | if (err) |
276 | return err; |
277 | |
278 | err = crypto_register_shash(alg: &crc32c_alg); |
279 | if (err) |
280 | return err; |
281 | |
282 | return 0; |
283 | } |
284 | |
285 | static void __exit crc32_mod_exit(void) |
286 | { |
287 | if (!cpu_has(CPU_FEATURE_CRC32)) |
288 | return; |
289 | |
290 | crypto_unregister_shash(alg: &crc32_alg); |
291 | crypto_unregister_shash(alg: &crc32c_alg); |
292 | } |
293 | |
294 | module_init(crc32_mod_init); |
295 | module_exit(crc32_mod_exit); |
296 | |
297 | MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>" ); |
298 | MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>" ); |
299 | MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions" ); |
300 | MODULE_LICENSE("GPL v2" ); |
301 | |