1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (C) 2023 SiFive |
4 | * Author: Andy Chiu <andy.chiu@sifive.com> |
5 | */ |
6 | #include <linux/export.h> |
7 | #include <linux/sched/signal.h> |
8 | #include <linux/types.h> |
9 | #include <linux/slab.h> |
10 | #include <linux/sched.h> |
11 | #include <linux/uaccess.h> |
12 | #include <linux/prctl.h> |
13 | |
14 | #include <asm/thread_info.h> |
15 | #include <asm/processor.h> |
16 | #include <asm/insn.h> |
17 | #include <asm/vector.h> |
18 | #include <asm/csr.h> |
19 | #include <asm/elf.h> |
20 | #include <asm/ptrace.h> |
21 | #include <asm/bug.h> |
22 | |
23 | static bool riscv_v_implicit_uacc = IS_ENABLED(CONFIG_RISCV_ISA_V_DEFAULT_ENABLE); |
24 | static struct kmem_cache *riscv_v_user_cachep; |
25 | #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE |
26 | static struct kmem_cache *riscv_v_kernel_cachep; |
27 | #endif |
28 | |
29 | unsigned long riscv_v_vsize __read_mostly; |
30 | EXPORT_SYMBOL_GPL(riscv_v_vsize); |
31 | |
32 | int riscv_v_setup_vsize(void) |
33 | { |
34 | unsigned long this_vsize; |
35 | |
36 | /* There are 32 vector registers with vlenb length. */ |
37 | riscv_v_enable(); |
38 | this_vsize = csr_read(CSR_VLENB) * 32; |
39 | riscv_v_disable(); |
40 | |
41 | if (!riscv_v_vsize) { |
42 | riscv_v_vsize = this_vsize; |
43 | return 0; |
44 | } |
45 | |
46 | if (riscv_v_vsize != this_vsize) { |
47 | WARN(1, "RISCV_ISA_V only supports one vlenb on SMP systems" ); |
48 | return -EOPNOTSUPP; |
49 | } |
50 | |
51 | return 0; |
52 | } |
53 | |
54 | void __init riscv_v_setup_ctx_cache(void) |
55 | { |
56 | if (!has_vector()) |
57 | return; |
58 | |
59 | riscv_v_user_cachep = kmem_cache_create_usercopy(name: "riscv_vector_ctx" , |
60 | size: riscv_v_vsize, align: 16, SLAB_PANIC, |
61 | useroffset: 0, usersize: riscv_v_vsize, NULL); |
62 | #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE |
63 | riscv_v_kernel_cachep = kmem_cache_create("riscv_vector_kctx" , |
64 | riscv_v_vsize, 16, |
65 | SLAB_PANIC, NULL); |
66 | #endif |
67 | } |
68 | |
69 | static bool insn_is_vector(u32 insn_buf) |
70 | { |
71 | u32 opcode = insn_buf & __INSN_OPCODE_MASK; |
72 | u32 width, csr; |
73 | |
74 | /* |
75 | * All V-related instructions, including CSR operations are 4-Byte. So, |
76 | * do not handle if the instruction length is not 4-Byte. |
77 | */ |
78 | if (unlikely(GET_INSN_LENGTH(insn_buf) != 4)) |
79 | return false; |
80 | |
81 | switch (opcode) { |
82 | case RVV_OPCODE_VECTOR: |
83 | return true; |
84 | case RVV_OPCODE_VL: |
85 | case RVV_OPCODE_VS: |
86 | width = RVV_EXRACT_VL_VS_WIDTH(insn_buf); |
87 | if (width == RVV_VL_VS_WIDTH_8 || width == RVV_VL_VS_WIDTH_16 || |
88 | width == RVV_VL_VS_WIDTH_32 || width == RVV_VL_VS_WIDTH_64) |
89 | return true; |
90 | |
91 | break; |
92 | case RVG_OPCODE_SYSTEM: |
93 | csr = RVG_EXTRACT_SYSTEM_CSR(insn_buf); |
94 | if ((csr >= CSR_VSTART && csr <= CSR_VCSR) || |
95 | (csr >= CSR_VL && csr <= CSR_VLENB)) |
96 | return true; |
97 | } |
98 | |
99 | return false; |
100 | } |
101 | |
102 | static int riscv_v_thread_zalloc(struct kmem_cache *cache, |
103 | struct __riscv_v_ext_state *ctx) |
104 | { |
105 | void *datap; |
106 | |
107 | datap = kmem_cache_zalloc(k: cache, GFP_KERNEL); |
108 | if (!datap) |
109 | return -ENOMEM; |
110 | |
111 | ctx->datap = datap; |
112 | memset(ctx, 0, offsetof(struct __riscv_v_ext_state, datap)); |
113 | return 0; |
114 | } |
115 | |
116 | void riscv_v_thread_alloc(struct task_struct *tsk) |
117 | { |
118 | #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE |
119 | riscv_v_thread_zalloc(riscv_v_kernel_cachep, &tsk->thread.kernel_vstate); |
120 | #endif |
121 | } |
122 | |
123 | void riscv_v_thread_free(struct task_struct *tsk) |
124 | { |
125 | if (tsk->thread.vstate.datap) |
126 | kmem_cache_free(s: riscv_v_user_cachep, objp: tsk->thread.vstate.datap); |
127 | #ifdef CONFIG_RISCV_ISA_V_PREEMPTIVE |
128 | if (tsk->thread.kernel_vstate.datap) |
129 | kmem_cache_free(riscv_v_kernel_cachep, tsk->thread.kernel_vstate.datap); |
130 | #endif |
131 | } |
132 | |
133 | #define VSTATE_CTRL_GET_CUR(x) ((x) & PR_RISCV_V_VSTATE_CTRL_CUR_MASK) |
134 | #define VSTATE_CTRL_GET_NEXT(x) (((x) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) >> 2) |
135 | #define VSTATE_CTRL_MAKE_NEXT(x) (((x) << 2) & PR_RISCV_V_VSTATE_CTRL_NEXT_MASK) |
136 | #define VSTATE_CTRL_GET_INHERIT(x) (!!((x) & PR_RISCV_V_VSTATE_CTRL_INHERIT)) |
137 | static inline int riscv_v_ctrl_get_cur(struct task_struct *tsk) |
138 | { |
139 | return VSTATE_CTRL_GET_CUR(tsk->thread.vstate_ctrl); |
140 | } |
141 | |
142 | static inline int riscv_v_ctrl_get_next(struct task_struct *tsk) |
143 | { |
144 | return VSTATE_CTRL_GET_NEXT(tsk->thread.vstate_ctrl); |
145 | } |
146 | |
147 | static inline bool riscv_v_ctrl_test_inherit(struct task_struct *tsk) |
148 | { |
149 | return VSTATE_CTRL_GET_INHERIT(tsk->thread.vstate_ctrl); |
150 | } |
151 | |
152 | static inline void riscv_v_ctrl_set(struct task_struct *tsk, int cur, int nxt, |
153 | bool inherit) |
154 | { |
155 | unsigned long ctrl; |
156 | |
157 | ctrl = cur & PR_RISCV_V_VSTATE_CTRL_CUR_MASK; |
158 | ctrl |= VSTATE_CTRL_MAKE_NEXT(nxt); |
159 | if (inherit) |
160 | ctrl |= PR_RISCV_V_VSTATE_CTRL_INHERIT; |
161 | tsk->thread.vstate_ctrl &= ~PR_RISCV_V_VSTATE_CTRL_MASK; |
162 | tsk->thread.vstate_ctrl |= ctrl; |
163 | } |
164 | |
165 | bool riscv_v_vstate_ctrl_user_allowed(void) |
166 | { |
167 | return riscv_v_ctrl_get_cur(current) == PR_RISCV_V_VSTATE_CTRL_ON; |
168 | } |
169 | EXPORT_SYMBOL_GPL(riscv_v_vstate_ctrl_user_allowed); |
170 | |
171 | bool riscv_v_first_use_handler(struct pt_regs *regs) |
172 | { |
173 | u32 __user *epc = (u32 __user *)regs->epc; |
174 | u32 insn = (u32)regs->badaddr; |
175 | |
176 | /* Do not handle if V is not supported, or disabled */ |
177 | if (!(ELF_HWCAP & COMPAT_HWCAP_ISA_V)) |
178 | return false; |
179 | |
180 | /* If V has been enabled then it is not the first-use trap */ |
181 | if (riscv_v_vstate_query(regs)) |
182 | return false; |
183 | |
184 | /* Get the instruction */ |
185 | if (!insn) { |
186 | if (__get_user(insn, epc)) |
187 | return false; |
188 | } |
189 | |
190 | /* Filter out non-V instructions */ |
191 | if (!insn_is_vector(insn_buf: insn)) |
192 | return false; |
193 | |
194 | /* Sanity check. datap should be null by the time of the first-use trap */ |
195 | WARN_ON(current->thread.vstate.datap); |
196 | |
197 | /* |
198 | * Now we sure that this is a V instruction. And it executes in the |
199 | * context where VS has been off. So, try to allocate the user's V |
200 | * context and resume execution. |
201 | */ |
202 | if (riscv_v_thread_zalloc(cache: riscv_v_user_cachep, ctx: ¤t->thread.vstate)) { |
203 | force_sig(SIGBUS); |
204 | return true; |
205 | } |
206 | riscv_v_vstate_on(regs); |
207 | riscv_v_vstate_set_restore(current, regs); |
208 | return true; |
209 | } |
210 | |
211 | void riscv_v_vstate_ctrl_init(struct task_struct *tsk) |
212 | { |
213 | bool inherit; |
214 | int cur, next; |
215 | |
216 | if (!has_vector()) |
217 | return; |
218 | |
219 | next = riscv_v_ctrl_get_next(tsk); |
220 | if (!next) { |
221 | if (READ_ONCE(riscv_v_implicit_uacc)) |
222 | cur = PR_RISCV_V_VSTATE_CTRL_ON; |
223 | else |
224 | cur = PR_RISCV_V_VSTATE_CTRL_OFF; |
225 | } else { |
226 | cur = next; |
227 | } |
228 | /* Clear next mask if inherit-bit is not set */ |
229 | inherit = riscv_v_ctrl_test_inherit(tsk); |
230 | if (!inherit) |
231 | next = PR_RISCV_V_VSTATE_CTRL_DEFAULT; |
232 | |
233 | riscv_v_ctrl_set(tsk, cur, nxt: next, inherit); |
234 | } |
235 | |
236 | long riscv_v_vstate_ctrl_get_current(void) |
237 | { |
238 | if (!has_vector()) |
239 | return -EINVAL; |
240 | |
241 | return current->thread.vstate_ctrl & PR_RISCV_V_VSTATE_CTRL_MASK; |
242 | } |
243 | |
244 | long riscv_v_vstate_ctrl_set_current(unsigned long arg) |
245 | { |
246 | bool inherit; |
247 | int cur, next; |
248 | |
249 | if (!has_vector()) |
250 | return -EINVAL; |
251 | |
252 | if (arg & ~PR_RISCV_V_VSTATE_CTRL_MASK) |
253 | return -EINVAL; |
254 | |
255 | cur = VSTATE_CTRL_GET_CUR(arg); |
256 | switch (cur) { |
257 | case PR_RISCV_V_VSTATE_CTRL_OFF: |
258 | /* Do not allow user to turn off V if current is not off */ |
259 | if (riscv_v_ctrl_get_cur(current) != PR_RISCV_V_VSTATE_CTRL_OFF) |
260 | return -EPERM; |
261 | |
262 | break; |
263 | case PR_RISCV_V_VSTATE_CTRL_ON: |
264 | break; |
265 | case PR_RISCV_V_VSTATE_CTRL_DEFAULT: |
266 | cur = riscv_v_ctrl_get_cur(current); |
267 | break; |
268 | default: |
269 | return -EINVAL; |
270 | } |
271 | |
272 | next = VSTATE_CTRL_GET_NEXT(arg); |
273 | inherit = VSTATE_CTRL_GET_INHERIT(arg); |
274 | switch (next) { |
275 | case PR_RISCV_V_VSTATE_CTRL_DEFAULT: |
276 | case PR_RISCV_V_VSTATE_CTRL_OFF: |
277 | case PR_RISCV_V_VSTATE_CTRL_ON: |
278 | riscv_v_ctrl_set(current, cur, nxt: next, inherit); |
279 | return 0; |
280 | } |
281 | |
282 | return -EINVAL; |
283 | } |
284 | |
285 | #ifdef CONFIG_SYSCTL |
286 | |
287 | static struct ctl_table riscv_v_default_vstate_table[] = { |
288 | { |
289 | .procname = "riscv_v_default_allow" , |
290 | .data = &riscv_v_implicit_uacc, |
291 | .maxlen = sizeof(riscv_v_implicit_uacc), |
292 | .mode = 0644, |
293 | .proc_handler = proc_dobool, |
294 | }, |
295 | }; |
296 | |
297 | static int __init riscv_v_sysctl_init(void) |
298 | { |
299 | if (has_vector()) |
300 | if (!register_sysctl("abi" , riscv_v_default_vstate_table)) |
301 | return -EINVAL; |
302 | return 0; |
303 | } |
304 | |
305 | #else /* ! CONFIG_SYSCTL */ |
306 | static int __init riscv_v_sysctl_init(void) { return 0; } |
307 | #endif /* ! CONFIG_SYSCTL */ |
308 | |
309 | static int riscv_v_init(void) |
310 | { |
311 | return riscv_v_sysctl_init(); |
312 | } |
313 | core_initcall(riscv_v_init); |
314 | |