1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * SM4 Cipher Algorithm, AES-NI/AVX2 optimized.
4 * as specified in
5 * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-10.html
6 *
7 * Copyright (c) 2021, Alibaba Group.
8 * Copyright (c) 2021 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
9 */
10
11#include <linux/module.h>
12#include <linux/crypto.h>
13#include <linux/kernel.h>
14#include <asm/simd.h>
15#include <crypto/internal/simd.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/sm4.h>
18#include "sm4-avx.h"
19
20#define SM4_CRYPT16_BLOCK_SIZE (SM4_BLOCK_SIZE * 16)
21
22asmlinkage void sm4_aesni_avx2_ctr_enc_blk16(const u32 *rk, u8 *dst,
23 const u8 *src, u8 *iv);
24asmlinkage void sm4_aesni_avx2_cbc_dec_blk16(const u32 *rk, u8 *dst,
25 const u8 *src, u8 *iv);
26asmlinkage void sm4_aesni_avx2_cfb_dec_blk16(const u32 *rk, u8 *dst,
27 const u8 *src, u8 *iv);
28
29static int sm4_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
30 unsigned int key_len)
31{
32 struct sm4_ctx *ctx = crypto_skcipher_ctx(tfm);
33
34 return sm4_expandkey(ctx, in_key: key, key_len);
35}
36
37static int cbc_decrypt(struct skcipher_request *req)
38{
39 return sm4_avx_cbc_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
40 func: sm4_aesni_avx2_cbc_dec_blk16);
41}
42
43
44static int cfb_decrypt(struct skcipher_request *req)
45{
46 return sm4_avx_cfb_decrypt(req, SM4_CRYPT16_BLOCK_SIZE,
47 func: sm4_aesni_avx2_cfb_dec_blk16);
48}
49
50static int ctr_crypt(struct skcipher_request *req)
51{
52 return sm4_avx_ctr_crypt(req, SM4_CRYPT16_BLOCK_SIZE,
53 func: sm4_aesni_avx2_ctr_enc_blk16);
54}
55
56static struct skcipher_alg sm4_aesni_avx2_skciphers[] = {
57 {
58 .base = {
59 .cra_name = "__ecb(sm4)",
60 .cra_driver_name = "__ecb-sm4-aesni-avx2",
61 .cra_priority = 500,
62 .cra_flags = CRYPTO_ALG_INTERNAL,
63 .cra_blocksize = SM4_BLOCK_SIZE,
64 .cra_ctxsize = sizeof(struct sm4_ctx),
65 .cra_module = THIS_MODULE,
66 },
67 .min_keysize = SM4_KEY_SIZE,
68 .max_keysize = SM4_KEY_SIZE,
69 .walksize = 16 * SM4_BLOCK_SIZE,
70 .setkey = sm4_skcipher_setkey,
71 .encrypt = sm4_avx_ecb_encrypt,
72 .decrypt = sm4_avx_ecb_decrypt,
73 }, {
74 .base = {
75 .cra_name = "__cbc(sm4)",
76 .cra_driver_name = "__cbc-sm4-aesni-avx2",
77 .cra_priority = 500,
78 .cra_flags = CRYPTO_ALG_INTERNAL,
79 .cra_blocksize = SM4_BLOCK_SIZE,
80 .cra_ctxsize = sizeof(struct sm4_ctx),
81 .cra_module = THIS_MODULE,
82 },
83 .min_keysize = SM4_KEY_SIZE,
84 .max_keysize = SM4_KEY_SIZE,
85 .ivsize = SM4_BLOCK_SIZE,
86 .walksize = 16 * SM4_BLOCK_SIZE,
87 .setkey = sm4_skcipher_setkey,
88 .encrypt = sm4_cbc_encrypt,
89 .decrypt = cbc_decrypt,
90 }, {
91 .base = {
92 .cra_name = "__cfb(sm4)",
93 .cra_driver_name = "__cfb-sm4-aesni-avx2",
94 .cra_priority = 500,
95 .cra_flags = CRYPTO_ALG_INTERNAL,
96 .cra_blocksize = 1,
97 .cra_ctxsize = sizeof(struct sm4_ctx),
98 .cra_module = THIS_MODULE,
99 },
100 .min_keysize = SM4_KEY_SIZE,
101 .max_keysize = SM4_KEY_SIZE,
102 .ivsize = SM4_BLOCK_SIZE,
103 .chunksize = SM4_BLOCK_SIZE,
104 .walksize = 16 * SM4_BLOCK_SIZE,
105 .setkey = sm4_skcipher_setkey,
106 .encrypt = sm4_cfb_encrypt,
107 .decrypt = cfb_decrypt,
108 }, {
109 .base = {
110 .cra_name = "__ctr(sm4)",
111 .cra_driver_name = "__ctr-sm4-aesni-avx2",
112 .cra_priority = 500,
113 .cra_flags = CRYPTO_ALG_INTERNAL,
114 .cra_blocksize = 1,
115 .cra_ctxsize = sizeof(struct sm4_ctx),
116 .cra_module = THIS_MODULE,
117 },
118 .min_keysize = SM4_KEY_SIZE,
119 .max_keysize = SM4_KEY_SIZE,
120 .ivsize = SM4_BLOCK_SIZE,
121 .chunksize = SM4_BLOCK_SIZE,
122 .walksize = 16 * SM4_BLOCK_SIZE,
123 .setkey = sm4_skcipher_setkey,
124 .encrypt = ctr_crypt,
125 .decrypt = ctr_crypt,
126 }
127};
128
129static struct simd_skcipher_alg *
130simd_sm4_aesni_avx2_skciphers[ARRAY_SIZE(sm4_aesni_avx2_skciphers)];
131
132static int __init sm4_init(void)
133{
134 const char *feature_name;
135
136 if (!boot_cpu_has(X86_FEATURE_AVX) ||
137 !boot_cpu_has(X86_FEATURE_AVX2) ||
138 !boot_cpu_has(X86_FEATURE_AES) ||
139 !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
140 pr_info("AVX2 or AES-NI instructions are not detected.\n");
141 return -ENODEV;
142 }
143
144 if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
145 feature_name: &feature_name)) {
146 pr_info("CPU feature '%s' is not supported.\n", feature_name);
147 return -ENODEV;
148 }
149
150 return simd_register_skciphers_compat(algs: sm4_aesni_avx2_skciphers,
151 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
152 simd_algs: simd_sm4_aesni_avx2_skciphers);
153}
154
155static void __exit sm4_exit(void)
156{
157 simd_unregister_skciphers(algs: sm4_aesni_avx2_skciphers,
158 ARRAY_SIZE(sm4_aesni_avx2_skciphers),
159 simd_algs: simd_sm4_aesni_avx2_skciphers);
160}
161
162module_init(sm4_init);
163module_exit(sm4_exit);
164
165MODULE_LICENSE("GPL v2");
166MODULE_AUTHOR("Tianjia Zhang <tianjia.zhang@linux.alibaba.com>");
167MODULE_DESCRIPTION("SM4 Cipher Algorithm, AES-NI/AVX2 optimized");
168MODULE_ALIAS_CRYPTO("sm4");
169MODULE_ALIAS_CRYPTO("sm4-aesni-avx2");
170

source code of linux/arch/x86/crypto/sm4_aesni_avx2_glue.c