1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <asm/fpu/api.h>
12#include <linux/module.h>
13#include <linux/crypto.h>
14#include <linux/kernel.h>
15#include <crypto/internal/skcipher.h>
16#include <crypto/sm4.h>
17#include "sm4-avx.h"
18
19#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
20
21asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
22 const u8 *src, u8 *iv);
23asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
24 const u8 *src, u8 *iv);
25
26static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
27 unsigned int key_len)
28{
29 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
30
31 return sm4_expandkey(ctx, in_key: key, key_len);
32}
33
34static int cbc_decrypt(struct skcipher_request *req)
35{
36 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
37 func: sm4_aesni_avx2_cbc_dec_blk16);
38}
39
40
41static int ctr_crypt(struct skcipher_request *req)
42{
43 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
44 func: sm4_aesni_avx2_ctr_enc_blk16);
45}
46
47static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
48 {
49 .base = {
50 .cra_name = "ecb(sm4)",
51 .cra_driver_name = "ecb-sm4-aesni-avx2",
52 .cra_priority = 500,
53 .cra_blocksize = SM4_BLOCK_SIZE,
54 .cra_ctxsize = sizeof(struct sm4_ctx),
55 .cra_module = THIS_MODULE,
56 },
57 .min_keysize = SM4_KEY_SIZE,
58 .max_keysize = SM4_KEY_SIZE,
59 .walksize = 16 * SM4_BLOCK_SIZE,
60 .setkey = sm4_skcipher_setkey,
61 .encrypt = sm4_avx_ecb_encrypt,
62 .decrypt = sm4_avx_ecb_decrypt,
63 }, {
64 .base = {
65 .cra_name = "cbc(sm4)",
66 .cra_driver_name = "cbc-sm4-aesni-avx2",
67 .cra_priority = 500,
68 .cra_blocksize = SM4_BLOCK_SIZE,
69 .cra_ctxsize = sizeof(struct sm4_ctx),
70 .cra_module = THIS_MODULE,
71 },
72 .min_keysize = SM4_KEY_SIZE,
73 .max_keysize = SM4_KEY_SIZE,
74 .ivsize = SM4_BLOCK_SIZE,
75 .walksize = 16 * SM4_BLOCK_SIZE,
76 .setkey = sm4_skcipher_setkey,
77 .encrypt = sm4_cbc_encrypt,
78 .decrypt = cbc_decrypt,
79 }, {
80 .base = {
81 .cra_name = "ctr(sm4)",
82 .cra_driver_name = "ctr-sm4-aesni-avx2",
83 .cra_priority = 500,
84 .cra_blocksize = 1,
85 .cra_ctxsize = sizeof(struct sm4_ctx),
86 .cra_module = THIS_MODULE,
87 },
88 .min_keysize = SM4_KEY_SIZE,
89 .max_keysize = SM4_KEY_SIZE,
90 .ivsize = SM4_BLOCK_SIZE,
91 .chunksize = SM4_BLOCK_SIZE,
92 .walksize = 16 * SM4_BLOCK_SIZE,
93 .setkey = sm4_skcipher_setkey,
94 .encrypt = ctr_crypt,
95 .decrypt = ctr_crypt,
96 }
97};
98
99static int __init sm4_init(void)
100{
101 const char *feature_name;
102
103 if (!boot_cpu_has(X86_FEATURE_AVX) ||
104 !boot_cpu_has(X86_FEATURE_AVX2) ||
105 !boot_cpu_has(X86_FEATURE_AES) ||
106 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
107 pr_info("AVX2 or AES-NI instructions are not detected.\n");
108 return -ENODEV;
109 }
110
111 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
112 feature_name: &feature_name)) {
113 pr_info("CPU feature '%s' is not supported.\n", feature_name);
114 return -ENODEV;
115 }
116
117 return crypto_register_skciphers(algs: sm4_aesni_avx2_skciphers,
118 ARRAY_SIZE(sm4_aesni_avx2_skciphers));
119}
120
121static void __exit sm4_exit(void)
122{
123 crypto_unregister_skciphers(algs: sm4_aesni_avx2_skciphers,
124 ARRAY_SIZE(sm4_aesni_avx2_skciphers));
125}
126
127module_init(sm4_init);
128module_exit(sm4_exit);
129
130MODULE_LICENSE("GPL v2");
131MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
132MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
133MODULE_ALIAS_CRYPTO("sm4");
134MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
135

source code of linux/arch/x86/crypto/sm4_aesni_avx2_glue.c