1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions |
4 | * |
5 | * Copyright (C) 2016 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> |
6 | */ |
7 | |
8 | #include <linux/cpufeature.h> |
9 | #include <linux/crc-t10dif.h> |
10 | #include <linux/init.h> |
11 | #include <linux/kernel.h> |
12 | #include <linux/module.h> |
13 | #include <linux/string.h> |
14 | |
15 | #include <crypto/internal/hash.h> |
16 | #include <crypto/internal/simd.h> |
17 | |
18 | #include <asm/neon.h> |
19 | #include <asm/simd.h> |
20 | |
21 | #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U |
22 | |
23 | asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); |
24 | asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); |
25 | |
26 | static int crct10dif_init(struct shash_desc *desc) |
27 | { |
28 | u16 *crc = shash_desc_ctx(desc); |
29 | |
30 | *crc = 0; |
31 | return 0; |
32 | } |
33 | |
34 | static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, |
35 | unsigned int length) |
36 | { |
37 | u16 *crc = shash_desc_ctx(desc); |
38 | |
39 | if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { |
40 | do { |
41 | unsigned int chunk = length; |
42 | |
43 | if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) |
44 | chunk = SZ_4K; |
45 | |
46 | kernel_neon_begin(); |
47 | *crc = crc_t10dif_pmull_p8(init_crc: *crc, buf: data, len: chunk); |
48 | kernel_neon_end(); |
49 | data += chunk; |
50 | length -= chunk; |
51 | } while (length); |
52 | } else { |
53 | *crc = crc_t10dif_generic(crc: *crc, buffer: data, len: length); |
54 | } |
55 | |
56 | return 0; |
57 | } |
58 | |
59 | static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, |
60 | unsigned int length) |
61 | { |
62 | u16 *crc = shash_desc_ctx(desc); |
63 | |
64 | if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { |
65 | do { |
66 | unsigned int chunk = length; |
67 | |
68 | if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) |
69 | chunk = SZ_4K; |
70 | |
71 | kernel_neon_begin(); |
72 | *crc = crc_t10dif_pmull_p64(init_crc: *crc, buf: data, len: chunk); |
73 | kernel_neon_end(); |
74 | data += chunk; |
75 | length -= chunk; |
76 | } while (length); |
77 | } else { |
78 | *crc = crc_t10dif_generic(crc: *crc, buffer: data, len: length); |
79 | } |
80 | |
81 | return 0; |
82 | } |
83 | |
84 | static int crct10dif_final(struct shash_desc *desc, u8 *out) |
85 | { |
86 | u16 *crc = shash_desc_ctx(desc); |
87 | |
88 | *(u16 *)out = *crc; |
89 | return 0; |
90 | } |
91 | |
92 | static struct shash_alg crc_t10dif_alg[] = {{ |
93 | .digestsize = CRC_T10DIF_DIGEST_SIZE, |
94 | .init = crct10dif_init, |
95 | .update = crct10dif_update_pmull_p8, |
96 | .final = crct10dif_final, |
97 | .descsize = CRC_T10DIF_DIGEST_SIZE, |
98 | |
99 | .base.cra_name = "crct10dif" , |
100 | .base.cra_driver_name = "crct10dif-arm64-neon" , |
101 | .base.cra_priority = 100, |
102 | .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, |
103 | .base.cra_module = THIS_MODULE, |
104 | }, { |
105 | .digestsize = CRC_T10DIF_DIGEST_SIZE, |
106 | .init = crct10dif_init, |
107 | .update = crct10dif_update_pmull_p64, |
108 | .final = crct10dif_final, |
109 | .descsize = CRC_T10DIF_DIGEST_SIZE, |
110 | |
111 | .base.cra_name = "crct10dif" , |
112 | .base.cra_driver_name = "crct10dif-arm64-ce" , |
113 | .base.cra_priority = 200, |
114 | .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, |
115 | .base.cra_module = THIS_MODULE, |
116 | }}; |
117 | |
118 | static int __init crc_t10dif_mod_init(void) |
119 | { |
120 | if (cpu_have_named_feature(PMULL)) |
121 | return crypto_register_shashes(algs: crc_t10dif_alg, |
122 | ARRAY_SIZE(crc_t10dif_alg)); |
123 | else |
124 | /* only register the first array element */ |
125 | return crypto_register_shash(alg: crc_t10dif_alg); |
126 | } |
127 | |
128 | static void __exit crc_t10dif_mod_exit(void) |
129 | { |
130 | if (cpu_have_named_feature(PMULL)) |
131 | crypto_unregister_shashes(algs: crc_t10dif_alg, |
132 | ARRAY_SIZE(crc_t10dif_alg)); |
133 | else |
134 | crypto_unregister_shash(alg: crc_t10dif_alg); |
135 | } |
136 | |
137 | module_cpu_feature_match(ASIMD, crc_t10dif_mod_init); |
138 | module_exit(crc_t10dif_mod_exit); |
139 | |
140 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>" ); |
141 | MODULE_LICENSE("GPL v2" ); |
142 | MODULE_ALIAS_CRYPTO("crct10dif" ); |
143 | MODULE_ALIAS_CRYPTO("crct10dif-arm64-ce" ); |
144 | |