1 | // SPDX-License-Identifier: GPL-2.0 OR MIT |
2 | /* |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. |
4 | * |
5 | * Based on public domain code from Daniel J. Bernstein and Peter Schwabe. This |
6 | * began from SUPERCOP's curve25519/neon2/scalarmult.s, but has subsequently been |
7 | * manually reworked for use in kernel space. |
8 | */ |
9 | |
10 | #include <asm/hwcap.h> |
11 | #include <asm/neon.h> |
12 | #include <asm/simd.h> |
13 | #include <crypto/internal/kpp.h> |
14 | #include <crypto/internal/simd.h> |
15 | #include <linux/types.h> |
16 | #include <linux/module.h> |
17 | #include <linux/init.h> |
18 | #include <linux/jump_label.h> |
19 | #include <linux/scatterlist.h> |
20 | #include <crypto/curve25519.h> |
21 | |
22 | asmlinkage void curve25519_neon(u8 mypublic[CURVE25519_KEY_SIZE], |
23 | const u8 secret[CURVE25519_KEY_SIZE], |
24 | const u8 basepoint[CURVE25519_KEY_SIZE]); |
25 | |
26 | static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon); |
27 | |
28 | void curve25519_arch(u8 out[CURVE25519_KEY_SIZE], |
29 | const u8 scalar[CURVE25519_KEY_SIZE], |
30 | const u8 point[CURVE25519_KEY_SIZE]) |
31 | { |
32 | if (static_branch_likely(&have_neon) && crypto_simd_usable()) { |
33 | kernel_neon_begin(); |
34 | curve25519_neon(mypublic: out, secret: scalar, basepoint: point); |
35 | kernel_neon_end(); |
36 | } else { |
37 | curve25519_generic(out, scalar, point); |
38 | } |
39 | } |
40 | EXPORT_SYMBOL(curve25519_arch); |
41 | |
42 | void curve25519_base_arch(u8 pub[CURVE25519_KEY_SIZE], |
43 | const u8 secret[CURVE25519_KEY_SIZE]) |
44 | { |
45 | return curve25519_arch(pub, secret, curve25519_base_point); |
46 | } |
47 | EXPORT_SYMBOL(curve25519_base_arch); |
48 | |
49 | static int curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, |
50 | unsigned int len) |
51 | { |
52 | u8 *secret = kpp_tfm_ctx(tfm); |
53 | |
54 | if (!len) |
55 | curve25519_generate_secret(secret); |
56 | else if (len == CURVE25519_KEY_SIZE && |
57 | crypto_memneq(a: buf, b: curve25519_null_point, size: CURVE25519_KEY_SIZE)) |
58 | memcpy(secret, buf, CURVE25519_KEY_SIZE); |
59 | else |
60 | return -EINVAL; |
61 | return 0; |
62 | } |
63 | |
64 | static int curve25519_compute_value(struct kpp_request *req) |
65 | { |
66 | struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); |
67 | const u8 *secret = kpp_tfm_ctx(tfm); |
68 | u8 public_key[CURVE25519_KEY_SIZE]; |
69 | u8 buf[CURVE25519_KEY_SIZE]; |
70 | int copied, nbytes; |
71 | u8 const *bp; |
72 | |
73 | if (req->src) { |
74 | copied = sg_copy_to_buffer(sgl: req->src, |
75 | nents: sg_nents_for_len(sg: req->src, |
76 | len: CURVE25519_KEY_SIZE), |
77 | buf: public_key, buflen: CURVE25519_KEY_SIZE); |
78 | if (copied != CURVE25519_KEY_SIZE) |
79 | return -EINVAL; |
80 | bp = public_key; |
81 | } else { |
82 | bp = curve25519_base_point; |
83 | } |
84 | |
85 | curve25519_arch(buf, secret, bp); |
86 | |
87 | /* might want less than we've got */ |
88 | nbytes = min_t(size_t, CURVE25519_KEY_SIZE, req->dst_len); |
89 | copied = sg_copy_from_buffer(sgl: req->dst, nents: sg_nents_for_len(sg: req->dst, |
90 | len: nbytes), |
91 | buf, buflen: nbytes); |
92 | if (copied != nbytes) |
93 | return -EINVAL; |
94 | return 0; |
95 | } |
96 | |
97 | static unsigned int curve25519_max_size(struct crypto_kpp *tfm) |
98 | { |
99 | return CURVE25519_KEY_SIZE; |
100 | } |
101 | |
102 | static struct kpp_alg curve25519_alg = { |
103 | .base.cra_name = "curve25519" , |
104 | .base.cra_driver_name = "curve25519-neon" , |
105 | .base.cra_priority = 200, |
106 | .base.cra_module = THIS_MODULE, |
107 | .base.cra_ctxsize = CURVE25519_KEY_SIZE, |
108 | |
109 | .set_secret = curve25519_set_secret, |
110 | .generate_public_key = curve25519_compute_value, |
111 | .compute_shared_secret = curve25519_compute_value, |
112 | .max_size = curve25519_max_size, |
113 | }; |
114 | |
115 | static int __init arm_curve25519_init(void) |
116 | { |
117 | if (elf_hwcap & HWCAP_NEON) { |
118 | static_branch_enable(&have_neon); |
119 | return IS_REACHABLE(CONFIG_CRYPTO_KPP) ? |
120 | crypto_register_kpp(alg: &curve25519_alg) : 0; |
121 | } |
122 | return 0; |
123 | } |
124 | |
125 | static void __exit arm_curve25519_exit(void) |
126 | { |
127 | if (IS_REACHABLE(CONFIG_CRYPTO_KPP) && elf_hwcap & HWCAP_NEON) |
128 | crypto_unregister_kpp(alg: &curve25519_alg); |
129 | } |
130 | |
131 | module_init(arm_curve25519_init); |
132 | module_exit(arm_curve25519_exit); |
133 | |
134 | MODULE_ALIAS_CRYPTO("curve25519" ); |
135 | MODULE_ALIAS_CRYPTO("curve25519-neon" ); |
136 | MODULE_LICENSE("GPL v2" ); |
137 | |