1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Accelerated CRC-T10DIF using ARM NEON and Crypto Extensions instructions |
4 | * |
5 | * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> |
6 | */ |
7 | |
8 | #include <linux/crc-t10dif.h> |
9 | #include <linux/init.h> |
10 | #include <linux/kernel.h> |
11 | #include <linux/module.h> |
12 | #include <linux/string.h> |
13 | |
14 | #include <crypto/internal/hash.h> |
15 | #include <crypto/internal/simd.h> |
16 | |
17 | #include <asm/neon.h> |
18 | #include <asm/simd.h> |
19 | |
20 | #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U |
21 | |
22 | asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); |
23 | |
24 | static int crct10dif_init(struct shash_desc *desc) |
25 | { |
26 | u16 *crc = shash_desc_ctx(desc); |
27 | |
28 | *crc = 0; |
29 | return 0; |
30 | } |
31 | |
32 | static int crct10dif_update(struct shash_desc *desc, const u8 *data, |
33 | unsigned int length) |
34 | { |
35 | u16 *crc = shash_desc_ctx(desc); |
36 | |
37 | if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { |
38 | kernel_neon_begin(); |
39 | *crc = crc_t10dif_pmull(init_crc: *crc, buf: data, len: length); |
40 | kernel_neon_end(); |
41 | } else { |
42 | *crc = crc_t10dif_generic(crc: *crc, buffer: data, len: length); |
43 | } |
44 | |
45 | return 0; |
46 | } |
47 | |
48 | static int crct10dif_final(struct shash_desc *desc, u8 *out) |
49 | { |
50 | u16 *crc = shash_desc_ctx(desc); |
51 | |
52 | *(u16 *)out = *crc; |
53 | return 0; |
54 | } |
55 | |
56 | static struct shash_alg crc_t10dif_alg = { |
57 | .digestsize = CRC_T10DIF_DIGEST_SIZE, |
58 | .init = crct10dif_init, |
59 | .update = crct10dif_update, |
60 | .final = crct10dif_final, |
61 | .descsize = CRC_T10DIF_DIGEST_SIZE, |
62 | |
63 | .base.cra_name = "crct10dif" , |
64 | .base.cra_driver_name = "crct10dif-arm-ce" , |
65 | .base.cra_priority = 200, |
66 | .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, |
67 | .base.cra_module = THIS_MODULE, |
68 | }; |
69 | |
70 | static int __init crc_t10dif_mod_init(void) |
71 | { |
72 | if (!(elf_hwcap2 & HWCAP2_PMULL)) |
73 | return -ENODEV; |
74 | |
75 | return crypto_register_shash(alg: &crc_t10dif_alg); |
76 | } |
77 | |
78 | static void __exit crc_t10dif_mod_exit(void) |
79 | { |
80 | crypto_unregister_shash(alg: &crc_t10dif_alg); |
81 | } |
82 | |
83 | module_init(crc_t10dif_mod_init); |
84 | module_exit(crc_t10dif_mod_exit); |
85 | |
86 | MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>" ); |
87 | MODULE_LICENSE("GPL v2" ); |
88 | MODULE_ALIAS_CRYPTO("crct10dif" ); |
89 | |