1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Support for AES-NI and VAES instructions. This file contains glue code.
4 * The real AES implementations are in aesni-intel_asm.S and other .S files.
5 *
6 * Copyright (C) 2008, Intel Corp.
7 * Author: Huang Ying <ying.huang@intel.com>
8 *
9 * Added RFC4106 AES-GCM support for 128-bit keys under the AEAD
10 * interface for 64-bit kernels.
11 * Authors: Adrian Hoban <adrian.hoban@intel.com>
12 * Gabriele Paoloni <gabriele.paoloni@intel.com>
13 * Tadeusz Struk (tadeusz.struk@intel.com)
14 * Aidan O'Mahony (aidan.o.mahony@intel.com)
15 * Copyright (c) 2010, Intel Corporation.
16 *
17 * Copyright 2024 Google LLC
18 */
19
20#include <linux/hardirq.h>
21#include <linux/types.h>
22#include <linux/module.h>
23#include <linux/err.h>
24#include <crypto/algapi.h>
25#include <crypto/aes.h>
26#include <crypto/b128ops.h>
27#include <crypto/gcm.h>
28#include <crypto/xts.h>
29#include <asm/cpu_device_id.h>
30#include <asm/simd.h>
31#include <crypto/scatterwalk.h>
32#include <crypto/internal/aead.h>
33#include <crypto/internal/simd.h>
34#include <crypto/internal/skcipher.h>
35#include <linux/jump_label.h>
36#include <linux/workqueue.h>
37#include <linux/spinlock.h>
38#include <linux/static_call.h>
39
40
41#define AESNI_ALIGN 16
42#define AESNI_ALIGN_ATTR __attribute__ ((__aligned__(AESNI_ALIGN)))
43#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE - 1))
44#define AESNI_ALIGN_EXTRA ((AESNI_ALIGN - 1) & ~(CRYPTO_MINALIGN - 1))
45#define CRYPTO_AES_CTX_SIZE (sizeof(struct crypto_aes_ctx) + AESNI_ALIGN_EXTRA)
46#define XTS_AES_CTX_SIZE (sizeof(struct aesni_xts_ctx) + AESNI_ALIGN_EXTRA)
47
48struct aesni_xts_ctx {
49 struct crypto_aes_ctx tweak_ctx AESNI_ALIGN_ATTR;
50 struct crypto_aes_ctx crypt_ctx AESNI_ALIGN_ATTR;
51};
52
53static inline void *aes_align_addr(void *addr)
54{
55 if (crypto_tfm_ctx_alignment() >= AESNI_ALIGN)
56 return addr;
57 return PTR_ALIGN(addr, AESNI_ALIGN);
58}
59
60asmlinkage void aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key,
61 unsigned int key_len);
62asmlinkage void aesni_enc(const void *ctx, u8 *out, const u8 *in);
63asmlinkage void aesni_dec(const void *ctx, u8 *out, const u8 *in);
64asmlinkage void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out,
65 const u8 *in, unsigned int len);
66asmlinkage void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out,
67 const u8 *in, unsigned int len);
68asmlinkage void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
69 const u8 *in, unsigned int len, u8 *iv);
70asmlinkage void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
71 const u8 *in, unsigned int len, u8 *iv);
72asmlinkage void aesni_cts_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out,
73 const u8 *in, unsigned int len, u8 *iv);
74asmlinkage void aesni_cts_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out,
75 const u8 *in, unsigned int len, u8 *iv);
76
77asmlinkage void aesni_xts_enc(const struct crypto_aes_ctx *ctx, u8 *out,
78 const u8 *in, unsigned int len, u8 *iv);
79
80asmlinkage void aesni_xts_dec(const struct crypto_aes_ctx *ctx, u8 *out,
81 const u8 *in, unsigned int len, u8 *iv);
82
83#ifdef CONFIG_X86_64
84asmlinkage void aesni_ctr_enc(struct crypto_aes_ctx *ctx, u8 *out,
85 const u8 *in, unsigned int len, u8 *iv);
86#endif
87
88static inline struct crypto_aes_ctx *aes_ctx(void *raw_ctx)
89{
90 return aes_align_addr(addr: raw_ctx);
91}
92
93static inline struct aesni_xts_ctx *aes_xts_ctx(struct crypto_skcipher *tfm)
94{
95 return aes_align_addr(addr: crypto_skcipher_ctx(tfm));
96}
97
98static int aes_set_key_common(struct crypto_aes_ctx *ctx,
99 const u8 *in_key, unsigned int key_len)
100{
101 int err;
102
103 if (!crypto_simd_usable())
104 return aes_expandkey(ctx, in_key, key_len);
105
106 err = aes_check_keylen(keylen: key_len);
107 if (err)
108 return err;
109
110 kernel_fpu_begin();
111 aesni_set_key(ctx, in_key, key_len);
112 kernel_fpu_end();
113 return 0;
114}
115
116static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
117 unsigned int key_len)
118{
119 return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_tfm_ctx(tfm)), in_key,
120 key_len);
121}
122
123static void aesni_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
124{
125 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm));
126
127 if (!crypto_simd_usable()) {
128 aes_encrypt(ctx, out: dst, in: src);
129 } else {
130 kernel_fpu_begin();
131 aesni_enc(ctx, out: dst, in: src);
132 kernel_fpu_end();
133 }
134}
135
136static void aesni_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src)
137{
138 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_tfm_ctx(tfm));
139
140 if (!crypto_simd_usable()) {
141 aes_decrypt(ctx, out: dst, in: src);
142 } else {
143 kernel_fpu_begin();
144 aesni_dec(ctx, out: dst, in: src);
145 kernel_fpu_end();
146 }
147}
148
149static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
150 unsigned int len)
151{
152 return aes_set_key_common(ctx: aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm)), in_key: key, key_len: len);
153}
154
155static int ecb_encrypt(struct skcipher_request *req)
156{
157 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
158 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
159 struct skcipher_walk walk;
160 unsigned int nbytes;
161 int err;
162
163 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
164
165 while ((nbytes = walk.nbytes)) {
166 kernel_fpu_begin();
167 aesni_ecb_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
168 len: nbytes & AES_BLOCK_MASK);
169 kernel_fpu_end();
170 nbytes &= AES_BLOCK_SIZE - 1;
171 err = skcipher_walk_done(walk: &walk, res: nbytes);
172 }
173
174 return err;
175}
176
177static int ecb_decrypt(struct skcipher_request *req)
178{
179 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
180 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
181 struct skcipher_walk walk;
182 unsigned int nbytes;
183 int err;
184
185 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
186
187 while ((nbytes = walk.nbytes)) {
188 kernel_fpu_begin();
189 aesni_ecb_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
190 len: nbytes & AES_BLOCK_MASK);
191 kernel_fpu_end();
192 nbytes &= AES_BLOCK_SIZE - 1;
193 err = skcipher_walk_done(walk: &walk, res: nbytes);
194 }
195
196 return err;
197}
198
199static int cbc_encrypt(struct skcipher_request *req)
200{
201 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
202 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
203 struct skcipher_walk walk;
204 unsigned int nbytes;
205 int err;
206
207 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
208
209 while ((nbytes = walk.nbytes)) {
210 kernel_fpu_begin();
211 aesni_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
212 len: nbytes & AES_BLOCK_MASK, iv: walk.iv);
213 kernel_fpu_end();
214 nbytes &= AES_BLOCK_SIZE - 1;
215 err = skcipher_walk_done(walk: &walk, res: nbytes);
216 }
217
218 return err;
219}
220
221static int cbc_decrypt(struct skcipher_request *req)
222{
223 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
224 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
225 struct skcipher_walk walk;
226 unsigned int nbytes;
227 int err;
228
229 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
230
231 while ((nbytes = walk.nbytes)) {
232 kernel_fpu_begin();
233 aesni_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
234 len: nbytes & AES_BLOCK_MASK, iv: walk.iv);
235 kernel_fpu_end();
236 nbytes &= AES_BLOCK_SIZE - 1;
237 err = skcipher_walk_done(walk: &walk, res: nbytes);
238 }
239
240 return err;
241}
242
243static int cts_cbc_encrypt(struct skcipher_request *req)
244{
245 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
246 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
247 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
248 struct scatterlist *src = req->src, *dst = req->dst;
249 struct scatterlist sg_src[2], sg_dst[2];
250 struct skcipher_request subreq;
251 struct skcipher_walk walk;
252 int err;
253
254 skcipher_request_set_tfm(req: &subreq, tfm);
255 skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req),
256 NULL, NULL);
257
258 if (req->cryptlen <= AES_BLOCK_SIZE) {
259 if (req->cryptlen < AES_BLOCK_SIZE)
260 return -EINVAL;
261 cbc_blocks = 1;
262 }
263
264 if (cbc_blocks > 0) {
265 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
266 cryptlen: cbc_blocks * AES_BLOCK_SIZE,
267 iv: req->iv);
268
269 err = cbc_encrypt(req: &subreq);
270 if (err)
271 return err;
272
273 if (req->cryptlen == AES_BLOCK_SIZE)
274 return 0;
275
276 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen);
277 if (req->dst != req->src)
278 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst,
279 len: subreq.cryptlen);
280 }
281
282 /* handle ciphertext stealing */
283 skcipher_request_set_crypt(req: &subreq, src, dst,
284 cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
285 iv: req->iv);
286
287 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
288 if (err)
289 return err;
290
291 kernel_fpu_begin();
292 aesni_cts_cbc_enc(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
293 len: walk.nbytes, iv: walk.iv);
294 kernel_fpu_end();
295
296 return skcipher_walk_done(walk: &walk, res: 0);
297}
298
299static int cts_cbc_decrypt(struct skcipher_request *req)
300{
301 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
302 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
303 int cbc_blocks = DIV_ROUND_UP(req->cryptlen, AES_BLOCK_SIZE) - 2;
304 struct scatterlist *src = req->src, *dst = req->dst;
305 struct scatterlist sg_src[2], sg_dst[2];
306 struct skcipher_request subreq;
307 struct skcipher_walk walk;
308 int err;
309
310 skcipher_request_set_tfm(req: &subreq, tfm);
311 skcipher_request_set_callback(req: &subreq, flags: skcipher_request_flags(req),
312 NULL, NULL);
313
314 if (req->cryptlen <= AES_BLOCK_SIZE) {
315 if (req->cryptlen < AES_BLOCK_SIZE)
316 return -EINVAL;
317 cbc_blocks = 1;
318 }
319
320 if (cbc_blocks > 0) {
321 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
322 cryptlen: cbc_blocks * AES_BLOCK_SIZE,
323 iv: req->iv);
324
325 err = cbc_decrypt(req: &subreq);
326 if (err)
327 return err;
328
329 if (req->cryptlen == AES_BLOCK_SIZE)
330 return 0;
331
332 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: subreq.cryptlen);
333 if (req->dst != req->src)
334 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst,
335 len: subreq.cryptlen);
336 }
337
338 /* handle ciphertext stealing */
339 skcipher_request_set_crypt(req: &subreq, src, dst,
340 cryptlen: req->cryptlen - cbc_blocks * AES_BLOCK_SIZE,
341 iv: req->iv);
342
343 err = skcipher_walk_virt(walk: &walk, req: &subreq, atomic: false);
344 if (err)
345 return err;
346
347 kernel_fpu_begin();
348 aesni_cts_cbc_dec(ctx, out: walk.dst.virt.addr, in: walk.src.virt.addr,
349 len: walk.nbytes, iv: walk.iv);
350 kernel_fpu_end();
351
352 return skcipher_walk_done(walk: &walk, res: 0);
353}
354
355#ifdef CONFIG_X86_64
356/* This is the non-AVX version. */
357static int ctr_crypt_aesni(struct skcipher_request *req)
358{
359 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
360 struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
361 u8 keystream[AES_BLOCK_SIZE];
362 struct skcipher_walk walk;
363 unsigned int nbytes;
364 int err;
365
366 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
367
368 while ((nbytes = walk.nbytes) > 0) {
369 kernel_fpu_begin();
370 if (nbytes & AES_BLOCK_MASK)
371 aesni_ctr_enc(ctx, out: walk.dst.virt.addr,
372 in: walk.src.virt.addr,
373 len: nbytes & AES_BLOCK_MASK, iv: walk.iv);
374 nbytes &= ~AES_BLOCK_MASK;
375
376 if (walk.nbytes == walk.total && nbytes > 0) {
377 aesni_enc(ctx, out: keystream, in: walk.iv);
378 crypto_xor_cpy(dst: walk.dst.virt.addr + walk.nbytes - nbytes,
379 src1: walk.src.virt.addr + walk.nbytes - nbytes,
380 src2: keystream, size: nbytes);
381 crypto_inc(a: walk.iv, AES_BLOCK_SIZE);
382 nbytes = 0;
383 }
384 kernel_fpu_end();
385 err = skcipher_walk_done(walk: &walk, res: nbytes);
386 }
387 return err;
388}
389#endif
390
391static int xts_setkey_aesni(struct crypto_skcipher *tfm, const u8 *key,
392 unsigned int keylen)
393{
394 struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
395 int err;
396
397 err = xts_verify_key(tfm, key, keylen);
398 if (err)
399 return err;
400
401 keylen /= 2;
402
403 /* first half of xts-key is for crypt */
404 err = aes_set_key_common(ctx: &ctx->crypt_ctx, in_key: key, key_len: keylen);
405 if (err)
406 return err;
407
408 /* second half of xts-key is for tweak */
409 return aes_set_key_common(ctx: &ctx->tweak_ctx, in_key: key + keylen, key_len: keylen);
410}
411
412typedef void (*xts_encrypt_iv_func)(const struct crypto_aes_ctx *tweak_key,
413 u8 iv[AES_BLOCK_SIZE]);
414typedef void (*xts_crypt_func)(const struct crypto_aes_ctx *key,
415 const u8 *src, u8 *dst, int len,
416 u8 tweak[AES_BLOCK_SIZE]);
417
418/* This handles cases where the source and/or destination span pages. */
419static noinline int
420xts_crypt_slowpath(struct skcipher_request *req, xts_crypt_func crypt_func)
421{
422 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
423 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
424 int tail = req->cryptlen % AES_BLOCK_SIZE;
425 struct scatterlist sg_src[2], sg_dst[2];
426 struct skcipher_request subreq;
427 struct skcipher_walk walk;
428 struct scatterlist *src, *dst;
429 int err;
430
431 /*
432 * If the message length isn't divisible by the AES block size, then
433 * separate off the last full block and the partial block. This ensures
434 * that they are processed in the same call to the assembly function,
435 * which is required for ciphertext stealing.
436 */
437 if (tail) {
438 skcipher_request_set_tfm(req: &subreq, tfm);
439 skcipher_request_set_callback(req: &subreq,
440 flags: skcipher_request_flags(req),
441 NULL, NULL);
442 skcipher_request_set_crypt(req: &subreq, src: req->src, dst: req->dst,
443 cryptlen: req->cryptlen - tail - AES_BLOCK_SIZE,
444 iv: req->iv);
445 req = &subreq;
446 }
447
448 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
449
450 while (walk.nbytes) {
451 kernel_fpu_begin();
452 (*crypt_func)(&ctx->crypt_ctx,
453 walk.src.virt.addr, walk.dst.virt.addr,
454 walk.nbytes & ~(AES_BLOCK_SIZE - 1), req->iv);
455 kernel_fpu_end();
456 err = skcipher_walk_done(walk: &walk,
457 res: walk.nbytes & (AES_BLOCK_SIZE - 1));
458 }
459
460 if (err || !tail)
461 return err;
462
463 /* Do ciphertext stealing with the last full block and partial block. */
464
465 dst = src = scatterwalk_ffwd(dst: sg_src, src: req->src, len: req->cryptlen);
466 if (req->dst != req->src)
467 dst = scatterwalk_ffwd(dst: sg_dst, src: req->dst, len: req->cryptlen);
468
469 skcipher_request_set_crypt(req, src, dst, AES_BLOCK_SIZE + tail,
470 iv: req->iv);
471
472 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
473 if (err)
474 return err;
475
476 kernel_fpu_begin();
477 (*crypt_func)(&ctx->crypt_ctx, walk.src.virt.addr, walk.dst.virt.addr,
478 walk.nbytes, req->iv);
479 kernel_fpu_end();
480
481 return skcipher_walk_done(walk: &walk, res: 0);
482}
483
484/* __always_inline to avoid indirect call in fastpath */
485static __always_inline int
486xts_crypt(struct skcipher_request *req, xts_encrypt_iv_func encrypt_iv,
487 xts_crypt_func crypt_func)
488{
489 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
490 const struct aesni_xts_ctx *ctx = aes_xts_ctx(tfm);
491
492 if (unlikely(req->cryptlen < AES_BLOCK_SIZE))
493 return -EINVAL;
494
495 kernel_fpu_begin();
496 (*encrypt_iv)(&ctx->tweak_ctx, req->iv);
497
498 /*
499 * In practice, virtually all XTS plaintexts and ciphertexts are either
500 * 512 or 4096 bytes and do not use multiple scatterlist elements. To
501 * optimize the performance of these cases, the below fast-path handles
502 * single-scatterlist-element messages as efficiently as possible. The
503 * code is 64-bit specific, as it assumes no page mapping is needed.
504 */
505 if (IS_ENABLED(CONFIG_X86_64) &&
506 likely(req->src->length >= req->cryptlen &&
507 req->dst->length >= req->cryptlen)) {
508 (*crypt_func)(&ctx->crypt_ctx, sg_virt(sg: req->src),
509 sg_virt(sg: req->dst), req->cryptlen, req->iv);
510 kernel_fpu_end();
511 return 0;
512 }
513 kernel_fpu_end();
514 return xts_crypt_slowpath(req, crypt_func);
515}
516
517static void aesni_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
518 u8 iv[AES_BLOCK_SIZE])
519{
520 aesni_enc(ctx: tweak_key, out: iv, in: iv);
521}
522
523static void aesni_xts_encrypt(const struct crypto_aes_ctx *key,
524 const u8 *src, u8 *dst, int len,
525 u8 tweak[AES_BLOCK_SIZE])
526{
527 aesni_xts_enc(ctx: key, out: dst, in: src, len, iv: tweak);
528}
529
530static void aesni_xts_decrypt(const struct crypto_aes_ctx *key,
531 const u8 *src, u8 *dst, int len,
532 u8 tweak[AES_BLOCK_SIZE])
533{
534 aesni_xts_dec(ctx: key, out: dst, in: src, len, iv: tweak);
535}
536
537static int xts_encrypt_aesni(struct skcipher_request *req)
538{
539 return xts_crypt(req, encrypt_iv: aesni_xts_encrypt_iv, crypt_func: aesni_xts_encrypt);
540}
541
542static int xts_decrypt_aesni(struct skcipher_request *req)
543{
544 return xts_crypt(req, encrypt_iv: aesni_xts_encrypt_iv, crypt_func: aesni_xts_decrypt);
545}
546
547static struct crypto_alg aesni_cipher_alg = {
548 .cra_name = "aes",
549 .cra_driver_name = "aes-aesni",
550 .cra_priority = 300,
551 .cra_flags = CRYPTO_ALG_TYPE_CIPHER,
552 .cra_blocksize = AES_BLOCK_SIZE,
553 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
554 .cra_module = THIS_MODULE,
555 .cra_u = {
556 .cipher = {
557 .cia_min_keysize = AES_MIN_KEY_SIZE,
558 .cia_max_keysize = AES_MAX_KEY_SIZE,
559 .cia_setkey = aes_set_key,
560 .cia_encrypt = aesni_encrypt,
561 .cia_decrypt = aesni_decrypt
562 }
563 }
564};
565
566static struct skcipher_alg aesni_skciphers[] = {
567 {
568 .base = {
569 .cra_name = "ecb(aes)",
570 .cra_driver_name = "ecb-aes-aesni",
571 .cra_priority = 400,
572 .cra_blocksize = AES_BLOCK_SIZE,
573 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
574 .cra_module = THIS_MODULE,
575 },
576 .min_keysize = AES_MIN_KEY_SIZE,
577 .max_keysize = AES_MAX_KEY_SIZE,
578 .setkey = aesni_skcipher_setkey,
579 .encrypt = ecb_encrypt,
580 .decrypt = ecb_decrypt,
581 }, {
582 .base = {
583 .cra_name = "cbc(aes)",
584 .cra_driver_name = "cbc-aes-aesni",
585 .cra_priority = 400,
586 .cra_blocksize = AES_BLOCK_SIZE,
587 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
588 .cra_module = THIS_MODULE,
589 },
590 .min_keysize = AES_MIN_KEY_SIZE,
591 .max_keysize = AES_MAX_KEY_SIZE,
592 .ivsize = AES_BLOCK_SIZE,
593 .setkey = aesni_skcipher_setkey,
594 .encrypt = cbc_encrypt,
595 .decrypt = cbc_decrypt,
596 }, {
597 .base = {
598 .cra_name = "cts(cbc(aes))",
599 .cra_driver_name = "cts-cbc-aes-aesni",
600 .cra_priority = 400,
601 .cra_blocksize = AES_BLOCK_SIZE,
602 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
603 .cra_module = THIS_MODULE,
604 },
605 .min_keysize = AES_MIN_KEY_SIZE,
606 .max_keysize = AES_MAX_KEY_SIZE,
607 .ivsize = AES_BLOCK_SIZE,
608 .walksize = 2 * AES_BLOCK_SIZE,
609 .setkey = aesni_skcipher_setkey,
610 .encrypt = cts_cbc_encrypt,
611 .decrypt = cts_cbc_decrypt,
612#ifdef CONFIG_X86_64
613 }, {
614 .base = {
615 .cra_name = "ctr(aes)",
616 .cra_driver_name = "ctr-aes-aesni",
617 .cra_priority = 400,
618 .cra_blocksize = 1,
619 .cra_ctxsize = CRYPTO_AES_CTX_SIZE,
620 .cra_module = THIS_MODULE,
621 },
622 .min_keysize = AES_MIN_KEY_SIZE,
623 .max_keysize = AES_MAX_KEY_SIZE,
624 .ivsize = AES_BLOCK_SIZE,
625 .chunksize = AES_BLOCK_SIZE,
626 .setkey = aesni_skcipher_setkey,
627 .encrypt = ctr_crypt_aesni,
628 .decrypt = ctr_crypt_aesni,
629#endif
630 }, {
631 .base = {
632 .cra_name = "xts(aes)",
633 .cra_driver_name = "xts-aes-aesni",
634 .cra_priority = 401,
635 .cra_blocksize = AES_BLOCK_SIZE,
636 .cra_ctxsize = XTS_AES_CTX_SIZE,
637 .cra_module = THIS_MODULE,
638 },
639 .min_keysize = 2 * AES_MIN_KEY_SIZE,
640 .max_keysize = 2 * AES_MAX_KEY_SIZE,
641 .ivsize = AES_BLOCK_SIZE,
642 .walksize = 2 * AES_BLOCK_SIZE,
643 .setkey = xts_setkey_aesni,
644 .encrypt = xts_encrypt_aesni,
645 .decrypt = xts_decrypt_aesni,
646 }
647};
648
649#ifdef CONFIG_X86_64
650asmlinkage void aes_xts_encrypt_iv(const struct crypto_aes_ctx *tweak_key,
651 u8 iv[AES_BLOCK_SIZE]);
652
653/* __always_inline to avoid indirect call */
654static __always_inline int
655ctr_crypt(struct skcipher_request *req,
656 void (*ctr64_func)(const struct crypto_aes_ctx *key,
657 const u8 *src, u8 *dst, int len,
658 const u64 le_ctr[2]))
659{
660 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
661 const struct crypto_aes_ctx *key = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
662 unsigned int nbytes, p1_nbytes, nblocks;
663 struct skcipher_walk walk;
664 u64 le_ctr[2];
665 u64 ctr64;
666 int err;
667
668 ctr64 = le_ctr[0] = get_unaligned_be64(p: &req->iv[8]);
669 le_ctr[1] = get_unaligned_be64(p: &req->iv[0]);
670
671 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
672
673 while ((nbytes = walk.nbytes) != 0) {
674 if (nbytes < walk.total) {
675 /* Not the end yet, so keep the length block-aligned. */
676 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
677 nblocks = nbytes / AES_BLOCK_SIZE;
678 } else {
679 /* It's the end, so include any final partial block. */
680 nblocks = DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
681 }
682 ctr64 += nblocks;
683
684 kernel_fpu_begin();
685 if (likely(ctr64 >= nblocks)) {
686 /* The low 64 bits of the counter won't overflow. */
687 (*ctr64_func)(key, walk.src.virt.addr,
688 walk.dst.virt.addr, nbytes, le_ctr);
689 } else {
690 /*
691 * The low 64 bits of the counter will overflow. The
692 * assembly doesn't handle this case, so split the
693 * operation into two at the point where the overflow
694 * will occur. After the first part, add the carry bit.
695 */
696 p1_nbytes = min(nbytes, (nblocks - ctr64) * AES_BLOCK_SIZE);
697 (*ctr64_func)(key, walk.src.virt.addr,
698 walk.dst.virt.addr, p1_nbytes, le_ctr);
699 le_ctr[0] = 0;
700 le_ctr[1]++;
701 (*ctr64_func)(key, walk.src.virt.addr + p1_nbytes,
702 walk.dst.virt.addr + p1_nbytes,
703 nbytes - p1_nbytes, le_ctr);
704 }
705 kernel_fpu_end();
706 le_ctr[0] = ctr64;
707
708 err = skcipher_walk_done(walk: &walk, res: walk.nbytes - nbytes);
709 }
710
711 put_unaligned_be64(val: ctr64, p: &req->iv[8]);
712 put_unaligned_be64(val: le_ctr[1], p: &req->iv[0]);
713
714 return err;
715}
716
717/* __always_inline to avoid indirect call */
718static __always_inline int
719xctr_crypt(struct skcipher_request *req,
720 void (*xctr_func)(const struct crypto_aes_ctx *key,
721 const u8 *src, u8 *dst, int len,
722 const u8 iv[AES_BLOCK_SIZE], u64 ctr))
723{
724 struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
725 const struct crypto_aes_ctx *key = aes_ctx(raw_ctx: crypto_skcipher_ctx(tfm));
726 struct skcipher_walk walk;
727 unsigned int nbytes;
728 u64 ctr = 1;
729 int err;
730
731 err = skcipher_walk_virt(walk: &walk, req, atomic: false);
732 while ((nbytes = walk.nbytes) != 0) {
733 if (nbytes < walk.total)
734 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
735
736 kernel_fpu_begin();
737 (*xctr_func)(key, walk.src.virt.addr, walk.dst.virt.addr,
738 nbytes, req->iv, ctr);
739 kernel_fpu_end();
740
741 ctr += DIV_ROUND_UP(nbytes, AES_BLOCK_SIZE);
742 err = skcipher_walk_done(walk: &walk, res: walk.nbytes - nbytes);
743 }
744 return err;
745}
746
747#define DEFINE_AVX_SKCIPHER_ALGS(suffix, driver_name_suffix, priority) \
748 \
749asmlinkage void \
750aes_xts_encrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
751 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
752asmlinkage void \
753aes_xts_decrypt_##suffix(const struct crypto_aes_ctx *key, const u8 *src, \
754 u8 *dst, int len, u8 tweak[AES_BLOCK_SIZE]); \
755 \
756static int xts_encrypt_##suffix(struct skcipher_request *req) \
757{ \
758 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_encrypt_##suffix); \
759} \
760 \
761static int xts_decrypt_##suffix(struct skcipher_request *req) \
762{ \
763 return xts_crypt(req, aes_xts_encrypt_iv, aes_xts_decrypt_##suffix); \
764} \
765 \
766asmlinkage void \
767aes_ctr64_crypt_##suffix(const struct crypto_aes_ctx *key, \
768 const u8 *src, u8 *dst, int len, const u64 le_ctr[2]);\
769 \
770static int ctr_crypt_##suffix(struct skcipher_request *req) \
771{ \
772 return ctr_crypt(req, aes_ctr64_crypt_##suffix); \
773} \
774 \
775asmlinkage void \
776aes_xctr_crypt_##suffix(const struct crypto_aes_ctx *key, \
777 const u8 *src, u8 *dst, int len, \
778 const u8 iv[AES_BLOCK_SIZE], u64 ctr); \
779 \
780static int xctr_crypt_##suffix(struct skcipher_request *req) \
781{ \
782 return xctr_crypt(req, aes_xctr_crypt_##suffix); \
783} \
784 \
785static struct skcipher_alg skcipher_algs_##suffix[] = {{ \
786 .base.cra_name = "xts(aes)", \
787 .base.cra_driver_name = "xts-aes-" driver_name_suffix, \
788 .base.cra_priority = priority, \
789 .base.cra_blocksize = AES_BLOCK_SIZE, \
790 .base.cra_ctxsize = XTS_AES_CTX_SIZE, \
791 .base.cra_module = THIS_MODULE, \
792 .min_keysize = 2 * AES_MIN_KEY_SIZE, \
793 .max_keysize = 2 * AES_MAX_KEY_SIZE, \
794 .ivsize = AES_BLOCK_SIZE, \
795 .walksize = 2 * AES_BLOCK_SIZE, \
796 .setkey = xts_setkey_aesni, \
797 .encrypt = xts_encrypt_##suffix, \
798 .decrypt = xts_decrypt_##suffix, \
799}, { \
800 .base.cra_name = "ctr(aes)", \
801 .base.cra_driver_name = "ctr-aes-" driver_name_suffix, \
802 .base.cra_priority = priority, \
803 .base.cra_blocksize = 1, \
804 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
805 .base.cra_module = THIS_MODULE, \
806 .min_keysize = AES_MIN_KEY_SIZE, \
807 .max_keysize = AES_MAX_KEY_SIZE, \
808 .ivsize = AES_BLOCK_SIZE, \
809 .chunksize = AES_BLOCK_SIZE, \
810 .setkey = aesni_skcipher_setkey, \
811 .encrypt = ctr_crypt_##suffix, \
812 .decrypt = ctr_crypt_##suffix, \
813}, { \
814 .base.cra_name = "xctr(aes)", \
815 .base.cra_driver_name = "xctr-aes-" driver_name_suffix, \
816 .base.cra_priority = priority, \
817 .base.cra_blocksize = 1, \
818 .base.cra_ctxsize = CRYPTO_AES_CTX_SIZE, \
819 .base.cra_module = THIS_MODULE, \
820 .min_keysize = AES_MIN_KEY_SIZE, \
821 .max_keysize = AES_MAX_KEY_SIZE, \
822 .ivsize = AES_BLOCK_SIZE, \
823 .chunksize = AES_BLOCK_SIZE, \
824 .setkey = aesni_skcipher_setkey, \
825 .encrypt = xctr_crypt_##suffix, \
826 .decrypt = xctr_crypt_##suffix, \
827}}
828
829DEFINE_AVX_SKCIPHER_ALGS(aesni_avx, "aesni-avx", 500);
830DEFINE_AVX_SKCIPHER_ALGS(vaes_avx2, "vaes-avx2", 600);
831DEFINE_AVX_SKCIPHER_ALGS(vaes_avx512, "vaes-avx512", 800);
832
833/* The common part of the x86_64 AES-GCM key struct */
834struct aes_gcm_key {
835 /* Expanded AES key and the AES key length in bytes */
836 struct crypto_aes_ctx aes_key;
837
838 /* RFC4106 nonce (used only by the rfc4106 algorithms) */
839 u32 rfc4106_nonce;
840};
841
842/* Key struct used by the AES-NI implementations of AES-GCM */
843struct aes_gcm_key_aesni {
844 /*
845 * Common part of the key. The assembly code requires 16-byte alignment
846 * for the round keys; we get this by them being located at the start of
847 * the struct and the whole struct being 16-byte aligned.
848 */
849 struct aes_gcm_key base;
850
851 /*
852 * Powers of the hash key H^8 through H^1. These are 128-bit values.
853 * They all have an extra factor of x^-1 and are byte-reversed. 16-byte
854 * alignment is required by the assembly code.
855 */
856 u64 h_powers[8][2] __aligned(16);
857
858 /*
859 * h_powers_xored[i] contains the two 64-bit halves of h_powers[i] XOR'd
860 * together. It's used for Karatsuba multiplication. 16-byte alignment
861 * is required by the assembly code.
862 */
863 u64 h_powers_xored[8] __aligned(16);
864
865 /*
866 * H^1 times x^64 (and also the usual extra factor of x^-1). 16-byte
867 * alignment is required by the assembly code.
868 */
869 u64 h_times_x64[2] __aligned(16);
870};
871#define AES_GCM_KEY_AESNI(key) \
872 container_of((key), struct aes_gcm_key_aesni, base)
873#define AES_GCM_KEY_AESNI_SIZE \
874 (sizeof(struct aes_gcm_key_aesni) + (15 & ~(CRYPTO_MINALIGN - 1)))
875
876/* Key struct used by the VAES + AVX2 implementation of AES-GCM */
877struct aes_gcm_key_vaes_avx2 {
878 /*
879 * Common part of the key. The assembly code prefers 16-byte alignment
880 * for the round keys; we get this by them being located at the start of
881 * the struct and the whole struct being 32-byte aligned.
882 */
883 struct aes_gcm_key base;
884
885 /*
886 * Powers of the hash key H^8 through H^1. These are 128-bit values.
887 * They all have an extra factor of x^-1 and are byte-reversed.
888 * The assembly code prefers 32-byte alignment for this.
889 */
890 u64 h_powers[8][2] __aligned(32);
891
892 /*
893 * Each entry in this array contains the two halves of an entry of
894 * h_powers XOR'd together, in the following order:
895 * H^8,H^6,H^7,H^5,H^4,H^2,H^3,H^1 i.e. indices 0,2,1,3,4,6,5,7.
896 * This is used for Karatsuba multiplication.
897 */
898 u64 h_powers_xored[8];
899};
900
901#define AES_GCM_KEY_VAES_AVX2(key) \
902 container_of((key), struct aes_gcm_key_vaes_avx2, base)
903#define AES_GCM_KEY_VAES_AVX2_SIZE \
904 (sizeof(struct aes_gcm_key_vaes_avx2) + (31 & ~(CRYPTO_MINALIGN - 1)))
905
906/* Key struct used by the VAES + AVX512 implementation of AES-GCM */
907struct aes_gcm_key_vaes_avx512 {
908 /*
909 * Common part of the key. The assembly code prefers 16-byte alignment
910 * for the round keys; we get this by them being located at the start of
911 * the struct and the whole struct being 64-byte aligned.
912 */
913 struct aes_gcm_key base;
914
915 /*
916 * Powers of the hash key H^16 through H^1. These are 128-bit values.
917 * They all have an extra factor of x^-1 and are byte-reversed. This
918 * array is aligned to a 64-byte boundary to make it naturally aligned
919 * for 512-bit loads, which can improve performance. (The assembly code
920 * doesn't *need* the alignment; this is just an optimization.)
921 */
922 u64 h_powers[16][2] __aligned(64);
923
924 /* Three padding blocks required by the assembly code */
925 u64 padding[3][2];
926};
927#define AES_GCM_KEY_VAES_AVX512(key) \
928 container_of((key), struct aes_gcm_key_vaes_avx512, base)
929#define AES_GCM_KEY_VAES_AVX512_SIZE \
930 (sizeof(struct aes_gcm_key_vaes_avx512) + (63 & ~(CRYPTO_MINALIGN - 1)))
931
932/*
933 * These flags are passed to the AES-GCM helper functions to specify the
934 * specific version of AES-GCM (RFC4106 or not), whether it's encryption or
935 * decryption, and which assembly functions should be called. Assembly
936 * functions are selected using flags instead of function pointers to avoid
937 * indirect calls (which are very expensive on x86) regardless of inlining.
938 */
939#define FLAG_RFC4106 BIT(0)
940#define FLAG_ENC BIT(1)
941#define FLAG_AVX BIT(2)
942#define FLAG_VAES_AVX2 BIT(3)
943#define FLAG_VAES_AVX512 BIT(4)
944
945static inline struct aes_gcm_key *
946aes_gcm_key_get(struct crypto_aead *tfm, int flags)
947{
948 if (flags & FLAG_VAES_AVX512)
949 return PTR_ALIGN(crypto_aead_ctx(tfm), 64);
950 else if (flags & FLAG_VAES_AVX2)
951 return PTR_ALIGN(crypto_aead_ctx(tfm), 32);
952 else
953 return PTR_ALIGN(crypto_aead_ctx(tfm), 16);
954}
955
956asmlinkage void
957aes_gcm_precompute_aesni(struct aes_gcm_key_aesni *key);
958asmlinkage void
959aes_gcm_precompute_aesni_avx(struct aes_gcm_key_aesni *key);
960asmlinkage void
961aes_gcm_precompute_vaes_avx2(struct aes_gcm_key_vaes_avx2 *key);
962asmlinkage void
963aes_gcm_precompute_vaes_avx512(struct aes_gcm_key_vaes_avx512 *key);
964
965static void aes_gcm_precompute(struct aes_gcm_key *key, int flags)
966{
967 if (flags & FLAG_VAES_AVX512)
968 aes_gcm_precompute_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key));
969 else if (flags & FLAG_VAES_AVX2)
970 aes_gcm_precompute_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key));
971 else if (flags & FLAG_AVX)
972 aes_gcm_precompute_aesni_avx(AES_GCM_KEY_AESNI(key));
973 else
974 aes_gcm_precompute_aesni(AES_GCM_KEY_AESNI(key));
975}
976
977asmlinkage void
978aes_gcm_aad_update_aesni(const struct aes_gcm_key_aesni *key,
979 u8 ghash_acc[16], const u8 *aad, int aadlen);
980asmlinkage void
981aes_gcm_aad_update_aesni_avx(const struct aes_gcm_key_aesni *key,
982 u8 ghash_acc[16], const u8 *aad, int aadlen);
983asmlinkage void
984aes_gcm_aad_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
985 u8 ghash_acc[16], const u8 *aad, int aadlen);
986asmlinkage void
987aes_gcm_aad_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
988 u8 ghash_acc[16], const u8 *aad, int aadlen);
989
990static void aes_gcm_aad_update(const struct aes_gcm_key *key, u8 ghash_acc[16],
991 const u8 *aad, int aadlen, int flags)
992{
993 if (flags & FLAG_VAES_AVX512)
994 aes_gcm_aad_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
995 ghash_acc, aad, aadlen);
996 else if (flags & FLAG_VAES_AVX2)
997 aes_gcm_aad_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
998 ghash_acc, aad, aadlen);
999 else if (flags & FLAG_AVX)
1000 aes_gcm_aad_update_aesni_avx(AES_GCM_KEY_AESNI(key), ghash_acc,
1001 aad, aadlen);
1002 else
1003 aes_gcm_aad_update_aesni(AES_GCM_KEY_AESNI(key), ghash_acc,
1004 aad, aadlen);
1005}
1006
1007asmlinkage void
1008aes_gcm_enc_update_aesni(const struct aes_gcm_key_aesni *key,
1009 const u32 le_ctr[4], u8 ghash_acc[16],
1010 const u8 *src, u8 *dst, int datalen);
1011asmlinkage void
1012aes_gcm_enc_update_aesni_avx(const struct aes_gcm_key_aesni *key,
1013 const u32 le_ctr[4], u8 ghash_acc[16],
1014 const u8 *src, u8 *dst, int datalen);
1015asmlinkage void
1016aes_gcm_enc_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1017 const u32 le_ctr[4], u8 ghash_acc[16],
1018 const u8 *src, u8 *dst, int datalen);
1019asmlinkage void
1020aes_gcm_enc_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1021 const u32 le_ctr[4], u8 ghash_acc[16],
1022 const u8 *src, u8 *dst, int datalen);
1023
1024asmlinkage void
1025aes_gcm_dec_update_aesni(const struct aes_gcm_key_aesni *key,
1026 const u32 le_ctr[4], u8 ghash_acc[16],
1027 const u8 *src, u8 *dst, int datalen);
1028asmlinkage void
1029aes_gcm_dec_update_aesni_avx(const struct aes_gcm_key_aesni *key,
1030 const u32 le_ctr[4], u8 ghash_acc[16],
1031 const u8 *src, u8 *dst, int datalen);
1032asmlinkage void
1033aes_gcm_dec_update_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1034 const u32 le_ctr[4], u8 ghash_acc[16],
1035 const u8 *src, u8 *dst, int datalen);
1036asmlinkage void
1037aes_gcm_dec_update_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1038 const u32 le_ctr[4], u8 ghash_acc[16],
1039 const u8 *src, u8 *dst, int datalen);
1040
1041/* __always_inline to optimize out the branches based on @flags */
1042static __always_inline void
1043aes_gcm_update(const struct aes_gcm_key *key,
1044 const u32 le_ctr[4], u8 ghash_acc[16],
1045 const u8 *src, u8 *dst, int datalen, int flags)
1046{
1047 if (flags & FLAG_ENC) {
1048 if (flags & FLAG_VAES_AVX512)
1049 aes_gcm_enc_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1050 le_ctr, ghash_acc,
1051 src, dst, datalen);
1052 else if (flags & FLAG_VAES_AVX2)
1053 aes_gcm_enc_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1054 le_ctr, ghash_acc,
1055 src, dst, datalen);
1056 else if (flags & FLAG_AVX)
1057 aes_gcm_enc_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1058 le_ctr, ghash_acc,
1059 src, dst, datalen);
1060 else
1061 aes_gcm_enc_update_aesni(AES_GCM_KEY_AESNI(key), le_ctr,
1062 ghash_acc, src, dst, datalen);
1063 } else {
1064 if (flags & FLAG_VAES_AVX512)
1065 aes_gcm_dec_update_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1066 le_ctr, ghash_acc,
1067 src, dst, datalen);
1068 else if (flags & FLAG_VAES_AVX2)
1069 aes_gcm_dec_update_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1070 le_ctr, ghash_acc,
1071 src, dst, datalen);
1072 else if (flags & FLAG_AVX)
1073 aes_gcm_dec_update_aesni_avx(AES_GCM_KEY_AESNI(key),
1074 le_ctr, ghash_acc,
1075 src, dst, datalen);
1076 else
1077 aes_gcm_dec_update_aesni(AES_GCM_KEY_AESNI(key),
1078 le_ctr, ghash_acc,
1079 src, dst, datalen);
1080 }
1081}
1082
1083asmlinkage void
1084aes_gcm_enc_final_aesni(const struct aes_gcm_key_aesni *key,
1085 const u32 le_ctr[4], u8 ghash_acc[16],
1086 u64 total_aadlen, u64 total_datalen);
1087asmlinkage void
1088aes_gcm_enc_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1089 const u32 le_ctr[4], u8 ghash_acc[16],
1090 u64 total_aadlen, u64 total_datalen);
1091asmlinkage void
1092aes_gcm_enc_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1093 const u32 le_ctr[4], u8 ghash_acc[16],
1094 u64 total_aadlen, u64 total_datalen);
1095asmlinkage void
1096aes_gcm_enc_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1097 const u32 le_ctr[4], u8 ghash_acc[16],
1098 u64 total_aadlen, u64 total_datalen);
1099
1100/* __always_inline to optimize out the branches based on @flags */
1101static __always_inline void
1102aes_gcm_enc_final(const struct aes_gcm_key *key,
1103 const u32 le_ctr[4], u8 ghash_acc[16],
1104 u64 total_aadlen, u64 total_datalen, int flags)
1105{
1106 if (flags & FLAG_VAES_AVX512)
1107 aes_gcm_enc_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1108 le_ctr, ghash_acc,
1109 total_aadlen, total_datalen);
1110 else if (flags & FLAG_VAES_AVX2)
1111 aes_gcm_enc_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1112 le_ctr, ghash_acc,
1113 total_aadlen, total_datalen);
1114 else if (flags & FLAG_AVX)
1115 aes_gcm_enc_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1116 le_ctr, ghash_acc,
1117 total_aadlen, total_datalen);
1118 else
1119 aes_gcm_enc_final_aesni(AES_GCM_KEY_AESNI(key),
1120 le_ctr, ghash_acc,
1121 total_aadlen, total_datalen);
1122}
1123
1124asmlinkage bool __must_check
1125aes_gcm_dec_final_aesni(const struct aes_gcm_key_aesni *key,
1126 const u32 le_ctr[4], const u8 ghash_acc[16],
1127 u64 total_aadlen, u64 total_datalen,
1128 const u8 tag[16], int taglen);
1129asmlinkage bool __must_check
1130aes_gcm_dec_final_aesni_avx(const struct aes_gcm_key_aesni *key,
1131 const u32 le_ctr[4], const u8 ghash_acc[16],
1132 u64 total_aadlen, u64 total_datalen,
1133 const u8 tag[16], int taglen);
1134asmlinkage bool __must_check
1135aes_gcm_dec_final_vaes_avx2(const struct aes_gcm_key_vaes_avx2 *key,
1136 const u32 le_ctr[4], const u8 ghash_acc[16],
1137 u64 total_aadlen, u64 total_datalen,
1138 const u8 tag[16], int taglen);
1139asmlinkage bool __must_check
1140aes_gcm_dec_final_vaes_avx512(const struct aes_gcm_key_vaes_avx512 *key,
1141 const u32 le_ctr[4], const u8 ghash_acc[16],
1142 u64 total_aadlen, u64 total_datalen,
1143 const u8 tag[16], int taglen);
1144
1145/* __always_inline to optimize out the branches based on @flags */
1146static __always_inline bool __must_check
1147aes_gcm_dec_final(const struct aes_gcm_key *key, const u32 le_ctr[4],
1148 u8 ghash_acc[16], u64 total_aadlen, u64 total_datalen,
1149 u8 tag[16], int taglen, int flags)
1150{
1151 if (flags & FLAG_VAES_AVX512)
1152 return aes_gcm_dec_final_vaes_avx512(AES_GCM_KEY_VAES_AVX512(key),
1153 le_ctr, ghash_acc,
1154 total_aadlen, total_datalen,
1155 tag, taglen);
1156 else if (flags & FLAG_VAES_AVX2)
1157 return aes_gcm_dec_final_vaes_avx2(AES_GCM_KEY_VAES_AVX2(key),
1158 le_ctr, ghash_acc,
1159 total_aadlen, total_datalen,
1160 tag, taglen);
1161 else if (flags & FLAG_AVX)
1162 return aes_gcm_dec_final_aesni_avx(AES_GCM_KEY_AESNI(key),
1163 le_ctr, ghash_acc,
1164 total_aadlen, total_datalen,
1165 tag, taglen);
1166 else
1167 return aes_gcm_dec_final_aesni(AES_GCM_KEY_AESNI(key),
1168 le_ctr, ghash_acc,
1169 total_aadlen, total_datalen,
1170 tag, taglen);
1171}
1172
1173/*
1174 * This is the Integrity Check Value (aka the authentication tag) length and can
1175 * be 8, 12 or 16 bytes long.
1176 */
1177static int common_rfc4106_set_authsize(struct crypto_aead *aead,
1178 unsigned int authsize)
1179{
1180 switch (authsize) {
1181 case 8:
1182 case 12:
1183 case 16:
1184 break;
1185 default:
1186 return -EINVAL;
1187 }
1188
1189 return 0;
1190}
1191
1192static int generic_gcmaes_set_authsize(struct crypto_aead *tfm,
1193 unsigned int authsize)
1194{
1195 switch (authsize) {
1196 case 4:
1197 case 8:
1198 case 12:
1199 case 13:
1200 case 14:
1201 case 15:
1202 case 16:
1203 break;
1204 default:
1205 return -EINVAL;
1206 }
1207
1208 return 0;
1209}
1210
1211/*
1212 * This is the setkey function for the x86_64 implementations of AES-GCM. It
1213 * saves the RFC4106 nonce if applicable, expands the AES key, and precomputes
1214 * powers of the hash key.
1215 *
1216 * To comply with the crypto_aead API, this has to be usable in no-SIMD context.
1217 * For that reason, this function includes a portable C implementation of the
1218 * needed logic. However, the portable C implementation is very slow, taking
1219 * about the same time as encrypting 37 KB of data. To be ready for users that
1220 * may set a key even somewhat frequently, we therefore also include a SIMD
1221 * assembly implementation, expanding the AES key using AES-NI and precomputing
1222 * the hash key powers using PCLMULQDQ or VPCLMULQDQ.
1223 */
1224static int gcm_setkey(struct crypto_aead *tfm, const u8 *raw_key,
1225 unsigned int keylen, int flags)
1226{
1227 struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1228 int err;
1229
1230 if (flags & FLAG_RFC4106) {
1231 if (keylen < 4)
1232 return -EINVAL;
1233 keylen -= 4;
1234 key->rfc4106_nonce = get_unaligned_be32(p: raw_key + keylen);
1235 }
1236
1237 /* The assembly code assumes the following offsets. */
1238 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_enc) != 0);
1239 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, base.aes_key.key_length) != 480);
1240 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers) != 496);
1241 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_powers_xored) != 624);
1242 BUILD_BUG_ON(offsetof(struct aes_gcm_key_aesni, h_times_x64) != 688);
1243 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_enc) != 0);
1244 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, base.aes_key.key_length) != 480);
1245 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers) != 512);
1246 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx2, h_powers_xored) != 640);
1247 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_enc) != 0);
1248 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, base.aes_key.key_length) != 480);
1249 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, h_powers) != 512);
1250 BUILD_BUG_ON(offsetof(struct aes_gcm_key_vaes_avx512, padding) != 768);
1251
1252 if (likely(crypto_simd_usable())) {
1253 err = aes_check_keylen(keylen);
1254 if (err)
1255 return err;
1256 kernel_fpu_begin();
1257 aesni_set_key(ctx: &key->aes_key, in_key: raw_key, key_len: keylen);
1258 aes_gcm_precompute(key, flags);
1259 kernel_fpu_end();
1260 } else {
1261 static const u8 x_to_the_minus1[16] __aligned(__alignof__(be128)) = {
1262 [0] = 0xc2, [15] = 1
1263 };
1264 static const u8 x_to_the_63[16] __aligned(__alignof__(be128)) = {
1265 [7] = 1,
1266 };
1267 be128 h1 = {};
1268 be128 h;
1269 int i;
1270
1271 err = aes_expandkey(ctx: &key->aes_key, in_key: raw_key, key_len: keylen);
1272 if (err)
1273 return err;
1274
1275 /* Encrypt the all-zeroes block to get the hash key H^1 */
1276 aes_encrypt(ctx: &key->aes_key, out: (u8 *)&h1, in: (u8 *)&h1);
1277
1278 /* Compute H^1 * x^-1 */
1279 h = h1;
1280 gf128mul_lle(a: &h, b: (const be128 *)x_to_the_minus1);
1281
1282 /* Compute the needed key powers */
1283 if (flags & FLAG_VAES_AVX512) {
1284 struct aes_gcm_key_vaes_avx512 *k =
1285 AES_GCM_KEY_VAES_AVX512(key);
1286
1287 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1288 k->h_powers[i][0] = be64_to_cpu(h.b);
1289 k->h_powers[i][1] = be64_to_cpu(h.a);
1290 gf128mul_lle(a: &h, b: &h1);
1291 }
1292 memset(k->padding, 0, sizeof(k->padding));
1293 } else if (flags & FLAG_VAES_AVX2) {
1294 struct aes_gcm_key_vaes_avx2 *k =
1295 AES_GCM_KEY_VAES_AVX2(key);
1296 static const u8 indices[8] = { 0, 2, 1, 3, 4, 6, 5, 7 };
1297
1298 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1299 k->h_powers[i][0] = be64_to_cpu(h.b);
1300 k->h_powers[i][1] = be64_to_cpu(h.a);
1301 gf128mul_lle(a: &h, b: &h1);
1302 }
1303 for (i = 0; i < ARRAY_SIZE(k->h_powers_xored); i++) {
1304 int j = indices[i];
1305
1306 k->h_powers_xored[i] = k->h_powers[j][0] ^
1307 k->h_powers[j][1];
1308 }
1309 } else {
1310 struct aes_gcm_key_aesni *k = AES_GCM_KEY_AESNI(key);
1311
1312 for (i = ARRAY_SIZE(k->h_powers) - 1; i >= 0; i--) {
1313 k->h_powers[i][0] = be64_to_cpu(h.b);
1314 k->h_powers[i][1] = be64_to_cpu(h.a);
1315 k->h_powers_xored[i] = k->h_powers[i][0] ^
1316 k->h_powers[i][1];
1317 gf128mul_lle(a: &h, b: &h1);
1318 }
1319 gf128mul_lle(a: &h1, b: (const be128 *)x_to_the_63);
1320 k->h_times_x64[0] = be64_to_cpu(h1.b);
1321 k->h_times_x64[1] = be64_to_cpu(h1.a);
1322 }
1323 }
1324 return 0;
1325}
1326
1327/*
1328 * Initialize @ghash_acc, then pass all @assoclen bytes of associated data
1329 * (a.k.a. additional authenticated data) from @sg_src through the GHASH update
1330 * assembly function. kernel_fpu_begin() must have already been called.
1331 */
1332static void gcm_process_assoc(const struct aes_gcm_key *key, u8 ghash_acc[16],
1333 struct scatterlist *sg_src, unsigned int assoclen,
1334 int flags)
1335{
1336 struct scatter_walk walk;
1337 /*
1338 * The assembly function requires that the length of any non-last
1339 * segment of associated data be a multiple of 16 bytes, so this
1340 * function does the buffering needed to achieve that.
1341 */
1342 unsigned int pos = 0;
1343 u8 buf[16];
1344
1345 memset(ghash_acc, 0, 16);
1346 scatterwalk_start(walk: &walk, sg: sg_src);
1347
1348 while (assoclen) {
1349 unsigned int orig_len_this_step = scatterwalk_next(
1350 walk: &walk, total: assoclen);
1351 unsigned int len_this_step = orig_len_this_step;
1352 unsigned int len;
1353 const u8 *src = walk.addr;
1354
1355 if (unlikely(pos)) {
1356 len = min(len_this_step, 16 - pos);
1357 memcpy(&buf[pos], src, len);
1358 pos += len;
1359 src += len;
1360 len_this_step -= len;
1361 if (pos < 16)
1362 goto next;
1363 aes_gcm_aad_update(key, ghash_acc, aad: buf, aadlen: 16, flags);
1364 pos = 0;
1365 }
1366 len = len_this_step;
1367 if (unlikely(assoclen)) /* Not the last segment yet? */
1368 len = round_down(len, 16);
1369 aes_gcm_aad_update(key, ghash_acc, aad: src, aadlen: len, flags);
1370 src += len;
1371 len_this_step -= len;
1372 if (unlikely(len_this_step)) {
1373 memcpy(buf, src, len_this_step);
1374 pos = len_this_step;
1375 }
1376next:
1377 scatterwalk_done_src(walk: &walk, nbytes: orig_len_this_step);
1378 if (need_resched()) {
1379 kernel_fpu_end();
1380 kernel_fpu_begin();
1381 }
1382 assoclen -= orig_len_this_step;
1383 }
1384 if (unlikely(pos))
1385 aes_gcm_aad_update(key, ghash_acc, aad: buf, aadlen: pos, flags);
1386}
1387
1388
1389/* __always_inline to optimize out the branches based on @flags */
1390static __always_inline int
1391gcm_crypt(struct aead_request *req, int flags)
1392{
1393 struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1394 const struct aes_gcm_key *key = aes_gcm_key_get(tfm, flags);
1395 unsigned int assoclen = req->assoclen;
1396 struct skcipher_walk walk;
1397 unsigned int nbytes;
1398 u8 ghash_acc[16]; /* GHASH accumulator */
1399 u32 le_ctr[4]; /* Counter in little-endian format */
1400 int taglen;
1401 int err;
1402
1403 /* Initialize the counter and determine the associated data length. */
1404 le_ctr[0] = 2;
1405 if (flags & FLAG_RFC4106) {
1406 if (unlikely(assoclen != 16 && assoclen != 20))
1407 return -EINVAL;
1408 assoclen -= 8;
1409 le_ctr[1] = get_unaligned_be32(p: req->iv + 4);
1410 le_ctr[2] = get_unaligned_be32(p: req->iv + 0);
1411 le_ctr[3] = key->rfc4106_nonce; /* already byte-swapped */
1412 } else {
1413 le_ctr[1] = get_unaligned_be32(p: req->iv + 8);
1414 le_ctr[2] = get_unaligned_be32(p: req->iv + 4);
1415 le_ctr[3] = get_unaligned_be32(p: req->iv + 0);
1416 }
1417
1418 /* Begin walking through the plaintext or ciphertext. */
1419 if (flags & FLAG_ENC)
1420 err = skcipher_walk_aead_encrypt(walk: &walk, req, atomic: false);
1421 else
1422 err = skcipher_walk_aead_decrypt(walk: &walk, req, atomic: false);
1423 if (err)
1424 return err;
1425
1426 /*
1427 * Since the AES-GCM assembly code requires that at least three assembly
1428 * functions be called to process any message (this is needed to support
1429 * incremental updates cleanly), to reduce overhead we try to do all
1430 * three calls in the same kernel FPU section if possible. We close the
1431 * section and start a new one if there are multiple data segments or if
1432 * rescheduling is needed while processing the associated data.
1433 */
1434 kernel_fpu_begin();
1435
1436 /* Pass the associated data through GHASH. */
1437 gcm_process_assoc(key, ghash_acc, sg_src: req->src, assoclen, flags);
1438
1439 /* En/decrypt the data and pass the ciphertext through GHASH. */
1440 while (unlikely((nbytes = walk.nbytes) < walk.total)) {
1441 /*
1442 * Non-last segment. In this case, the assembly function
1443 * requires that the length be a multiple of 16 (AES_BLOCK_SIZE)
1444 * bytes. The needed buffering of up to 16 bytes is handled by
1445 * the skcipher_walk. Here we just need to round down to a
1446 * multiple of 16.
1447 */
1448 nbytes = round_down(nbytes, AES_BLOCK_SIZE);
1449 aes_gcm_update(key, le_ctr, ghash_acc, src: walk.src.virt.addr,
1450 dst: walk.dst.virt.addr, datalen: nbytes, flags);
1451 le_ctr[0] += nbytes / AES_BLOCK_SIZE;
1452 kernel_fpu_end();
1453 err = skcipher_walk_done(walk: &walk, res: walk.nbytes - nbytes);
1454 if (err)
1455 return err;
1456 kernel_fpu_begin();
1457 }
1458 /* Last segment: process all remaining data. */
1459 aes_gcm_update(key, le_ctr, ghash_acc, src: walk.src.virt.addr,
1460 dst: walk.dst.virt.addr, datalen: nbytes, flags);
1461 /*
1462 * The low word of the counter isn't used by the finalize, so there's no
1463 * need to increment it here.
1464 */
1465
1466 /* Finalize */
1467 taglen = crypto_aead_authsize(tfm);
1468 if (flags & FLAG_ENC) {
1469 /* Finish computing the auth tag. */
1470 aes_gcm_enc_final(key, le_ctr, ghash_acc, total_aadlen: assoclen,
1471 total_datalen: req->cryptlen, flags);
1472
1473 /* Store the computed auth tag in the dst scatterlist. */
1474 scatterwalk_map_and_copy(buf: ghash_acc, sg: req->dst, start: req->assoclen +
1475 req->cryptlen, nbytes: taglen, out: 1);
1476 } else {
1477 unsigned int datalen = req->cryptlen - taglen;
1478 u8 tag[16];
1479
1480 /* Get the transmitted auth tag from the src scatterlist. */
1481 scatterwalk_map_and_copy(buf: tag, sg: req->src, start: req->assoclen + datalen,
1482 nbytes: taglen, out: 0);
1483 /*
1484 * Finish computing the auth tag and compare it to the
1485 * transmitted one. The assembly function does the actual tag
1486 * comparison. Here, just check the boolean result.
1487 */
1488 if (!aes_gcm_dec_final(key, le_ctr, ghash_acc, total_aadlen: assoclen,
1489 total_datalen: datalen, tag, taglen, flags))
1490 err = -EBADMSG;
1491 }
1492 kernel_fpu_end();
1493 if (nbytes)
1494 skcipher_walk_done(walk: &walk, res: 0);
1495 return err;
1496}
1497
1498#define DEFINE_GCM_ALGS(suffix, flags, generic_driver_name, rfc_driver_name, \
1499 ctxsize, priority) \
1500 \
1501static int gcm_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \
1502 unsigned int keylen) \
1503{ \
1504 return gcm_setkey(tfm, raw_key, keylen, (flags)); \
1505} \
1506 \
1507static int gcm_encrypt_##suffix(struct aead_request *req) \
1508{ \
1509 return gcm_crypt(req, (flags) | FLAG_ENC); \
1510} \
1511 \
1512static int gcm_decrypt_##suffix(struct aead_request *req) \
1513{ \
1514 return gcm_crypt(req, (flags)); \
1515} \
1516 \
1517static int rfc4106_setkey_##suffix(struct crypto_aead *tfm, const u8 *raw_key, \
1518 unsigned int keylen) \
1519{ \
1520 return gcm_setkey(tfm, raw_key, keylen, (flags) | FLAG_RFC4106); \
1521} \
1522 \
1523static int rfc4106_encrypt_##suffix(struct aead_request *req) \
1524{ \
1525 return gcm_crypt(req, (flags) | FLAG_RFC4106 | FLAG_ENC); \
1526} \
1527 \
1528static int rfc4106_decrypt_##suffix(struct aead_request *req) \
1529{ \
1530 return gcm_crypt(req, (flags) | FLAG_RFC4106); \
1531} \
1532 \
1533static struct aead_alg aes_gcm_algs_##suffix[] = { { \
1534 .setkey = gcm_setkey_##suffix, \
1535 .setauthsize = generic_gcmaes_set_authsize, \
1536 .encrypt = gcm_encrypt_##suffix, \
1537 .decrypt = gcm_decrypt_##suffix, \
1538 .ivsize = GCM_AES_IV_SIZE, \
1539 .chunksize = AES_BLOCK_SIZE, \
1540 .maxauthsize = 16, \
1541 .base = { \
1542 .cra_name = "gcm(aes)", \
1543 .cra_driver_name = generic_driver_name, \
1544 .cra_priority = (priority), \
1545 .cra_blocksize = 1, \
1546 .cra_ctxsize = (ctxsize), \
1547 .cra_module = THIS_MODULE, \
1548 }, \
1549}, { \
1550 .setkey = rfc4106_setkey_##suffix, \
1551 .setauthsize = common_rfc4106_set_authsize, \
1552 .encrypt = rfc4106_encrypt_##suffix, \
1553 .decrypt = rfc4106_decrypt_##suffix, \
1554 .ivsize = GCM_RFC4106_IV_SIZE, \
1555 .chunksize = AES_BLOCK_SIZE, \
1556 .maxauthsize = 16, \
1557 .base = { \
1558 .cra_name = "rfc4106(gcm(aes))", \
1559 .cra_driver_name = rfc_driver_name, \
1560 .cra_priority = (priority), \
1561 .cra_blocksize = 1, \
1562 .cra_ctxsize = (ctxsize), \
1563 .cra_module = THIS_MODULE, \
1564 }, \
1565} }
1566
1567/* aes_gcm_algs_aesni */
1568DEFINE_GCM_ALGS(aesni, /* no flags */ 0,
1569 "generic-gcm-aesni", "rfc4106-gcm-aesni",
1570 AES_GCM_KEY_AESNI_SIZE, 400);
1571
1572/* aes_gcm_algs_aesni_avx */
1573DEFINE_GCM_ALGS(aesni_avx, FLAG_AVX,
1574 "generic-gcm-aesni-avx", "rfc4106-gcm-aesni-avx",
1575 AES_GCM_KEY_AESNI_SIZE, 500);
1576
1577/* aes_gcm_algs_vaes_avx2 */
1578DEFINE_GCM_ALGS(vaes_avx2, FLAG_VAES_AVX2,
1579 "generic-gcm-vaes-avx2", "rfc4106-gcm-vaes-avx2",
1580 AES_GCM_KEY_VAES_AVX2_SIZE, 600);
1581
1582/* aes_gcm_algs_vaes_avx512 */
1583DEFINE_GCM_ALGS(vaes_avx512, FLAG_VAES_AVX512,
1584 "generic-gcm-vaes-avx512", "rfc4106-gcm-vaes-avx512",
1585 AES_GCM_KEY_VAES_AVX512_SIZE, 800);
1586
1587static int __init register_avx_algs(void)
1588{
1589 int err;
1590
1591 if (!boot_cpu_has(X86_FEATURE_AVX))
1592 return 0;
1593 err = crypto_register_skciphers(algs: skcipher_algs_aesni_avx,
1594 ARRAY_SIZE(skcipher_algs_aesni_avx));
1595 if (err)
1596 return err;
1597 err = crypto_register_aeads(algs: aes_gcm_algs_aesni_avx,
1598 ARRAY_SIZE(aes_gcm_algs_aesni_avx));
1599 if (err)
1600 return err;
1601 /*
1602 * Note: not all the algorithms registered below actually require
1603 * VPCLMULQDQ. But in practice every CPU with VAES also has VPCLMULQDQ.
1604 * Similarly, the assembler support was added at about the same time.
1605 * For simplicity, just always check for VAES and VPCLMULQDQ together.
1606 */
1607 if (!boot_cpu_has(X86_FEATURE_AVX2) ||
1608 !boot_cpu_has(X86_FEATURE_VAES) ||
1609 !boot_cpu_has(X86_FEATURE_VPCLMULQDQ) ||
1610 !boot_cpu_has(X86_FEATURE_PCLMULQDQ) ||
1611 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL))
1612 return 0;
1613 err = crypto_register_skciphers(algs: skcipher_algs_vaes_avx2,
1614 ARRAY_SIZE(skcipher_algs_vaes_avx2));
1615 if (err)
1616 return err;
1617 err = crypto_register_aeads(algs: aes_gcm_algs_vaes_avx2,
1618 ARRAY_SIZE(aes_gcm_algs_vaes_avx2));
1619 if (err)
1620 return err;
1621
1622 if (!boot_cpu_has(X86_FEATURE_AVX512BW) ||
1623 !boot_cpu_has(X86_FEATURE_AVX512VL) ||
1624 !boot_cpu_has(X86_FEATURE_BMI2) ||
1625 !cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM |
1626 XFEATURE_MASK_AVX512, NULL))
1627 return 0;
1628
1629 if (boot_cpu_has(X86_FEATURE_PREFER_YMM)) {
1630 int i;
1631
1632 for (i = 0; i < ARRAY_SIZE(skcipher_algs_vaes_avx512); i++)
1633 skcipher_algs_vaes_avx512[i].base.cra_priority = 1;
1634 for (i = 0; i < ARRAY_SIZE(aes_gcm_algs_vaes_avx512); i++)
1635 aes_gcm_algs_vaes_avx512[i].base.cra_priority = 1;
1636 }
1637
1638 err = crypto_register_skciphers(algs: skcipher_algs_vaes_avx512,
1639 ARRAY_SIZE(skcipher_algs_vaes_avx512));
1640 if (err)
1641 return err;
1642 err = crypto_register_aeads(algs: aes_gcm_algs_vaes_avx512,
1643 ARRAY_SIZE(aes_gcm_algs_vaes_avx512));
1644 if (err)
1645 return err;
1646
1647 return 0;
1648}
1649
1650#define unregister_skciphers(A) \
1651 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1652 crypto_unregister_skciphers((A), ARRAY_SIZE(A))
1653#define unregister_aeads(A) \
1654 if (refcount_read(&(A)[0].base.cra_refcnt) != 0) \
1655 crypto_unregister_aeads((A), ARRAY_SIZE(A))
1656
1657static void unregister_avx_algs(void)
1658{
1659 unregister_skciphers(skcipher_algs_aesni_avx);
1660 unregister_aeads(aes_gcm_algs_aesni_avx);
1661 unregister_skciphers(skcipher_algs_vaes_avx2);
1662 unregister_skciphers(skcipher_algs_vaes_avx512);
1663 unregister_aeads(aes_gcm_algs_vaes_avx2);
1664 unregister_aeads(aes_gcm_algs_vaes_avx512);
1665}
1666#else /* CONFIG_X86_64 */
1667static struct aead_alg aes_gcm_algs_aesni[0];
1668
1669static int __init register_avx_algs(void)
1670{
1671 return 0;
1672}
1673
1674static void unregister_avx_algs(void)
1675{
1676}
1677#endif /* !CONFIG_X86_64 */
1678
1679static const struct x86_cpu_id aesni_cpu_id[] = {
1680 X86_MATCH_FEATURE(X86_FEATURE_AES, NULL),
1681 {}
1682};
1683MODULE_DEVICE_TABLE(x86cpu, aesni_cpu_id);
1684
1685static int __init aesni_init(void)
1686{
1687 int err;
1688
1689 if (!x86_match_cpu(match: aesni_cpu_id))
1690 return -ENODEV;
1691
1692 err = crypto_register_alg(alg: &aesni_cipher_alg);
1693 if (err)
1694 return err;
1695
1696 err = crypto_register_skciphers(algs: aesni_skciphers,
1697 ARRAY_SIZE(aesni_skciphers));
1698 if (err)
1699 goto unregister_cipher;
1700
1701 err = crypto_register_aeads(algs: aes_gcm_algs_aesni,
1702 ARRAY_SIZE(aes_gcm_algs_aesni));
1703 if (err)
1704 goto unregister_skciphers;
1705
1706 err = register_avx_algs();
1707 if (err)
1708 goto unregister_avx;
1709
1710 return 0;
1711
1712unregister_avx:
1713 unregister_avx_algs();
1714 crypto_unregister_aeads(algs: aes_gcm_algs_aesni,
1715 ARRAY_SIZE(aes_gcm_algs_aesni));
1716unregister_skciphers:
1717 crypto_unregister_skciphers(algs: aesni_skciphers,
1718 ARRAY_SIZE(aesni_skciphers));
1719unregister_cipher:
1720 crypto_unregister_alg(alg: &aesni_cipher_alg);
1721 return err;
1722}
1723
1724static void __exit aesni_exit(void)
1725{
1726 crypto_unregister_aeads(algs: aes_gcm_algs_aesni,
1727 ARRAY_SIZE(aes_gcm_algs_aesni));
1728 crypto_unregister_skciphers(algs: aesni_skciphers,
1729 ARRAY_SIZE(aesni_skciphers));
1730 crypto_unregister_alg(alg: &aesni_cipher_alg);
1731 unregister_avx_algs();
1732}
1733
1734module_init(aesni_init);
1735module_exit(aesni_exit);
1736
1737MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions");
1738MODULE_LICENSE("GPL");
1739MODULE_ALIAS_CRYPTO("aes");
1740

source code of linux/arch/x86/crypto/aesni-intel_glue.c