1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
4 | * |
5 | * This is an implementation of the ChaCha20Poly1305 AEAD construction. |
6 | * |
7 | * Information: https://tools.ietf.org/html/rfc8439 |
8 | */ |
9 | |
10 | #include <crypto/algapi.h> |
11 | #include <crypto/chacha20poly1305.h> |
12 | #include <crypto/chacha.h> |
13 | #include <crypto/poly1305.h> |
14 | #include <crypto/scatterwalk.h> |
15 | |
16 | #include <asm/unaligned.h> |
17 | #include <linux/kernel.h> |
18 | #include <linux/init.h> |
19 | #include <linux/mm.h> |
20 | #include <linux/module.h> |
21 | |
22 | #define CHACHA_KEY_WORDS (CHACHA_KEY_SIZE / sizeof(u32)) |
23 | |
24 | static void chacha_load_key(u32 *k, const u8 *in) |
25 | { |
26 | k[0] = get_unaligned_le32(p: in); |
27 | k[1] = get_unaligned_le32(p: in + 4); |
28 | k[2] = get_unaligned_le32(p: in + 8); |
29 | k[3] = get_unaligned_le32(p: in + 12); |
30 | k[4] = get_unaligned_le32(p: in + 16); |
31 | k[5] = get_unaligned_le32(p: in + 20); |
32 | k[6] = get_unaligned_le32(p: in + 24); |
33 | k[7] = get_unaligned_le32(p: in + 28); |
34 | } |
35 | |
36 | static void xchacha_init(u32 *chacha_state, const u8 *key, const u8 *nonce) |
37 | { |
38 | u32 k[CHACHA_KEY_WORDS]; |
39 | u8 iv[CHACHA_IV_SIZE]; |
40 | |
41 | memset(iv, 0, 8); |
42 | memcpy(iv + 8, nonce + 16, 8); |
43 | |
44 | chacha_load_key(k, in: key); |
45 | |
46 | /* Compute the subkey given the original key and first 128 nonce bits */ |
47 | chacha_init(state: chacha_state, key: k, iv: nonce); |
48 | hchacha_block(state: chacha_state, out: k, nrounds: 20); |
49 | |
50 | chacha_init(state: chacha_state, key: k, iv); |
51 | |
52 | memzero_explicit(s: k, count: sizeof(k)); |
53 | memzero_explicit(s: iv, count: sizeof(iv)); |
54 | } |
55 | |
56 | static void |
57 | __chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, |
58 | const u8 *ad, const size_t ad_len, u32 *chacha_state) |
59 | { |
60 | const u8 *pad0 = page_address(ZERO_PAGE(0)); |
61 | struct poly1305_desc_ctx poly1305_state; |
62 | union { |
63 | u8 block0[POLY1305_KEY_SIZE]; |
64 | __le64 lens[2]; |
65 | } b; |
66 | |
67 | chacha20_crypt(state: chacha_state, dst: b.block0, src: pad0, bytes: sizeof(b.block0)); |
68 | poly1305_init(desc: &poly1305_state, key: b.block0); |
69 | |
70 | poly1305_update(desc: &poly1305_state, src: ad, nbytes: ad_len); |
71 | if (ad_len & 0xf) |
72 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (ad_len & 0xf)); |
73 | |
74 | chacha20_crypt(state: chacha_state, dst, src, bytes: src_len); |
75 | |
76 | poly1305_update(desc: &poly1305_state, src: dst, nbytes: src_len); |
77 | if (src_len & 0xf) |
78 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (src_len & 0xf)); |
79 | |
80 | b.lens[0] = cpu_to_le64(ad_len); |
81 | b.lens[1] = cpu_to_le64(src_len); |
82 | poly1305_update(desc: &poly1305_state, src: (u8 *)b.lens, nbytes: sizeof(b.lens)); |
83 | |
84 | poly1305_final(desc: &poly1305_state, digest: dst + src_len); |
85 | |
86 | memzero_explicit(s: chacha_state, CHACHA_STATE_WORDS * sizeof(u32)); |
87 | memzero_explicit(s: &b, count: sizeof(b)); |
88 | } |
89 | |
90 | void chacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, |
91 | const u8 *ad, const size_t ad_len, |
92 | const u64 nonce, |
93 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
94 | { |
95 | u32 chacha_state[CHACHA_STATE_WORDS]; |
96 | u32 k[CHACHA_KEY_WORDS]; |
97 | __le64 iv[2]; |
98 | |
99 | chacha_load_key(k, in: key); |
100 | |
101 | iv[0] = 0; |
102 | iv[1] = cpu_to_le64(nonce); |
103 | |
104 | chacha_init(state: chacha_state, key: k, iv: (u8 *)iv); |
105 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); |
106 | |
107 | memzero_explicit(s: iv, count: sizeof(iv)); |
108 | memzero_explicit(s: k, count: sizeof(k)); |
109 | } |
110 | EXPORT_SYMBOL(chacha20poly1305_encrypt); |
111 | |
112 | void xchacha20poly1305_encrypt(u8 *dst, const u8 *src, const size_t src_len, |
113 | const u8 *ad, const size_t ad_len, |
114 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], |
115 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
116 | { |
117 | u32 chacha_state[CHACHA_STATE_WORDS]; |
118 | |
119 | xchacha_init(chacha_state, key, nonce); |
120 | __chacha20poly1305_encrypt(dst, src, src_len, ad, ad_len, chacha_state); |
121 | } |
122 | EXPORT_SYMBOL(xchacha20poly1305_encrypt); |
123 | |
124 | static bool |
125 | __chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, |
126 | const u8 *ad, const size_t ad_len, u32 *chacha_state) |
127 | { |
128 | const u8 *pad0 = page_address(ZERO_PAGE(0)); |
129 | struct poly1305_desc_ctx poly1305_state; |
130 | size_t dst_len; |
131 | int ret; |
132 | union { |
133 | u8 block0[POLY1305_KEY_SIZE]; |
134 | u8 mac[POLY1305_DIGEST_SIZE]; |
135 | __le64 lens[2]; |
136 | } b; |
137 | |
138 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) |
139 | return false; |
140 | |
141 | chacha20_crypt(state: chacha_state, dst: b.block0, src: pad0, bytes: sizeof(b.block0)); |
142 | poly1305_init(desc: &poly1305_state, key: b.block0); |
143 | |
144 | poly1305_update(desc: &poly1305_state, src: ad, nbytes: ad_len); |
145 | if (ad_len & 0xf) |
146 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (ad_len & 0xf)); |
147 | |
148 | dst_len = src_len - POLY1305_DIGEST_SIZE; |
149 | poly1305_update(desc: &poly1305_state, src, nbytes: dst_len); |
150 | if (dst_len & 0xf) |
151 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (dst_len & 0xf)); |
152 | |
153 | b.lens[0] = cpu_to_le64(ad_len); |
154 | b.lens[1] = cpu_to_le64(dst_len); |
155 | poly1305_update(desc: &poly1305_state, src: (u8 *)b.lens, nbytes: sizeof(b.lens)); |
156 | |
157 | poly1305_final(desc: &poly1305_state, digest: b.mac); |
158 | |
159 | ret = crypto_memneq(a: b.mac, b: src + dst_len, POLY1305_DIGEST_SIZE); |
160 | if (likely(!ret)) |
161 | chacha20_crypt(state: chacha_state, dst, src, bytes: dst_len); |
162 | |
163 | memzero_explicit(s: &b, count: sizeof(b)); |
164 | |
165 | return !ret; |
166 | } |
167 | |
168 | bool chacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, |
169 | const u8 *ad, const size_t ad_len, |
170 | const u64 nonce, |
171 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
172 | { |
173 | u32 chacha_state[CHACHA_STATE_WORDS]; |
174 | u32 k[CHACHA_KEY_WORDS]; |
175 | __le64 iv[2]; |
176 | bool ret; |
177 | |
178 | chacha_load_key(k, in: key); |
179 | |
180 | iv[0] = 0; |
181 | iv[1] = cpu_to_le64(nonce); |
182 | |
183 | chacha_init(state: chacha_state, key: k, iv: (u8 *)iv); |
184 | ret = __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, |
185 | chacha_state); |
186 | |
187 | memzero_explicit(s: chacha_state, count: sizeof(chacha_state)); |
188 | memzero_explicit(s: iv, count: sizeof(iv)); |
189 | memzero_explicit(s: k, count: sizeof(k)); |
190 | return ret; |
191 | } |
192 | EXPORT_SYMBOL(chacha20poly1305_decrypt); |
193 | |
194 | bool xchacha20poly1305_decrypt(u8 *dst, const u8 *src, const size_t src_len, |
195 | const u8 *ad, const size_t ad_len, |
196 | const u8 nonce[XCHACHA20POLY1305_NONCE_SIZE], |
197 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
198 | { |
199 | u32 chacha_state[CHACHA_STATE_WORDS]; |
200 | |
201 | xchacha_init(chacha_state, key, nonce); |
202 | return __chacha20poly1305_decrypt(dst, src, src_len, ad, ad_len, |
203 | chacha_state); |
204 | } |
205 | EXPORT_SYMBOL(xchacha20poly1305_decrypt); |
206 | |
207 | static |
208 | bool chacha20poly1305_crypt_sg_inplace(struct scatterlist *src, |
209 | const size_t src_len, |
210 | const u8 *ad, const size_t ad_len, |
211 | const u64 nonce, |
212 | const u8 key[CHACHA20POLY1305_KEY_SIZE], |
213 | int encrypt) |
214 | { |
215 | const u8 *pad0 = page_address(ZERO_PAGE(0)); |
216 | struct poly1305_desc_ctx poly1305_state; |
217 | u32 chacha_state[CHACHA_STATE_WORDS]; |
218 | struct sg_mapping_iter miter; |
219 | size_t partial = 0; |
220 | unsigned int flags; |
221 | bool ret = true; |
222 | int sl; |
223 | union { |
224 | struct { |
225 | u32 k[CHACHA_KEY_WORDS]; |
226 | __le64 iv[2]; |
227 | }; |
228 | u8 block0[POLY1305_KEY_SIZE]; |
229 | u8 chacha_stream[CHACHA_BLOCK_SIZE]; |
230 | struct { |
231 | u8 mac[2][POLY1305_DIGEST_SIZE]; |
232 | }; |
233 | __le64 lens[2]; |
234 | } b __aligned(16); |
235 | |
236 | if (WARN_ON(src_len > INT_MAX)) |
237 | return false; |
238 | |
239 | chacha_load_key(k: b.k, in: key); |
240 | |
241 | b.iv[0] = 0; |
242 | b.iv[1] = cpu_to_le64(nonce); |
243 | |
244 | chacha_init(state: chacha_state, key: b.k, iv: (u8 *)b.iv); |
245 | chacha20_crypt(state: chacha_state, dst: b.block0, src: pad0, bytes: sizeof(b.block0)); |
246 | poly1305_init(desc: &poly1305_state, key: b.block0); |
247 | |
248 | if (unlikely(ad_len)) { |
249 | poly1305_update(desc: &poly1305_state, src: ad, nbytes: ad_len); |
250 | if (ad_len & 0xf) |
251 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (ad_len & 0xf)); |
252 | } |
253 | |
254 | flags = SG_MITER_TO_SG | SG_MITER_ATOMIC; |
255 | |
256 | sg_miter_start(miter: &miter, sgl: src, nents: sg_nents(sg: src), flags); |
257 | |
258 | for (sl = src_len; sl > 0 && sg_miter_next(miter: &miter); sl -= miter.length) { |
259 | u8 *addr = miter.addr; |
260 | size_t length = min_t(size_t, sl, miter.length); |
261 | |
262 | if (!encrypt) |
263 | poly1305_update(desc: &poly1305_state, src: addr, nbytes: length); |
264 | |
265 | if (unlikely(partial)) { |
266 | size_t l = min(length, CHACHA_BLOCK_SIZE - partial); |
267 | |
268 | crypto_xor(dst: addr, src: b.chacha_stream + partial, size: l); |
269 | partial = (partial + l) & (CHACHA_BLOCK_SIZE - 1); |
270 | |
271 | addr += l; |
272 | length -= l; |
273 | } |
274 | |
275 | if (likely(length >= CHACHA_BLOCK_SIZE || length == sl)) { |
276 | size_t l = length; |
277 | |
278 | if (unlikely(length < sl)) |
279 | l &= ~(CHACHA_BLOCK_SIZE - 1); |
280 | chacha20_crypt(state: chacha_state, dst: addr, src: addr, bytes: l); |
281 | addr += l; |
282 | length -= l; |
283 | } |
284 | |
285 | if (unlikely(length > 0)) { |
286 | chacha20_crypt(state: chacha_state, dst: b.chacha_stream, src: pad0, |
287 | CHACHA_BLOCK_SIZE); |
288 | crypto_xor(dst: addr, src: b.chacha_stream, size: length); |
289 | partial = length; |
290 | } |
291 | |
292 | if (encrypt) |
293 | poly1305_update(desc: &poly1305_state, src: miter.addr, |
294 | min_t(size_t, sl, miter.length)); |
295 | } |
296 | |
297 | if (src_len & 0xf) |
298 | poly1305_update(desc: &poly1305_state, src: pad0, nbytes: 0x10 - (src_len & 0xf)); |
299 | |
300 | b.lens[0] = cpu_to_le64(ad_len); |
301 | b.lens[1] = cpu_to_le64(src_len); |
302 | poly1305_update(desc: &poly1305_state, src: (u8 *)b.lens, nbytes: sizeof(b.lens)); |
303 | |
304 | if (likely(sl <= -POLY1305_DIGEST_SIZE)) { |
305 | if (encrypt) { |
306 | poly1305_final(desc: &poly1305_state, |
307 | digest: miter.addr + miter.length + sl); |
308 | ret = true; |
309 | } else { |
310 | poly1305_final(desc: &poly1305_state, digest: b.mac[0]); |
311 | ret = !crypto_memneq(a: b.mac[0], |
312 | b: miter.addr + miter.length + sl, |
313 | POLY1305_DIGEST_SIZE); |
314 | } |
315 | } |
316 | |
317 | sg_miter_stop(miter: &miter); |
318 | |
319 | if (unlikely(sl > -POLY1305_DIGEST_SIZE)) { |
320 | poly1305_final(desc: &poly1305_state, digest: b.mac[1]); |
321 | scatterwalk_map_and_copy(buf: b.mac[encrypt], sg: src, start: src_len, |
322 | nbytes: sizeof(b.mac[1]), out: encrypt); |
323 | ret = encrypt || |
324 | !crypto_memneq(a: b.mac[0], b: b.mac[1], POLY1305_DIGEST_SIZE); |
325 | } |
326 | |
327 | memzero_explicit(s: chacha_state, count: sizeof(chacha_state)); |
328 | memzero_explicit(s: &b, count: sizeof(b)); |
329 | |
330 | return ret; |
331 | } |
332 | |
333 | bool chacha20poly1305_encrypt_sg_inplace(struct scatterlist *src, size_t src_len, |
334 | const u8 *ad, const size_t ad_len, |
335 | const u64 nonce, |
336 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
337 | { |
338 | return chacha20poly1305_crypt_sg_inplace(src, src_len, ad, ad_len, |
339 | nonce, key, encrypt: 1); |
340 | } |
341 | EXPORT_SYMBOL(chacha20poly1305_encrypt_sg_inplace); |
342 | |
343 | bool chacha20poly1305_decrypt_sg_inplace(struct scatterlist *src, size_t src_len, |
344 | const u8 *ad, const size_t ad_len, |
345 | const u64 nonce, |
346 | const u8 key[CHACHA20POLY1305_KEY_SIZE]) |
347 | { |
348 | if (unlikely(src_len < POLY1305_DIGEST_SIZE)) |
349 | return false; |
350 | |
351 | return chacha20poly1305_crypt_sg_inplace(src, |
352 | src_len: src_len - POLY1305_DIGEST_SIZE, |
353 | ad, ad_len, nonce, key, encrypt: 0); |
354 | } |
355 | EXPORT_SYMBOL(chacha20poly1305_decrypt_sg_inplace); |
356 | |
357 | static int __init chacha20poly1305_init(void) |
358 | { |
359 | if (!IS_ENABLED(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) && |
360 | WARN_ON(!chacha20poly1305_selftest())) |
361 | return -ENODEV; |
362 | return 0; |
363 | } |
364 | |
365 | static void __exit chacha20poly1305_exit(void) |
366 | { |
367 | } |
368 | |
369 | module_init(chacha20poly1305_init); |
370 | module_exit(chacha20poly1305_exit); |
371 | MODULE_LICENSE("GPL v2" ); |
372 | MODULE_DESCRIPTION("ChaCha20Poly1305 AEAD construction" ); |
373 | MODULE_AUTHOR("Jason A. Donenfeld <Jason@zx2c4.com>" ); |
374 | |